sql

lib/sql/test/CDB_AnalysisCheckTest.sql

@@ -0,0 +1,20 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+SELECT CDB_SetUserQuotaInBytes(1000000);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int);
+SELECT _CDB_AnalysisTablesInSchema('public') t ORDER BY t;
+SELECT _CDB_AnalysisDataSize('public');
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+SELECT CDB_SetUserQuotaInBytes(1);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9;
+DROP FUNCTION "public"._CDB_UserQuotaInBytes();

lib/sql/test/CDB_AnalysisCheckTest_expect

@@ -0,0 +1,18 @@
+SET
+1000000
+0
+CREATE TABLE
+CREATE TABLE
+CREATE TABLE
+(analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5,public,analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5)
+(analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94,public,analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94)
+0
+
+1
+
+INSERT 0 5
+ERROR:  Analysis cache space limits exceeded
+DROP TABLE
+DROP TABLE
+DROP TABLE
+DROP FUNCTION

lib/sql/test/CDB_CartodbfyTableTest.sql

@@ -0,0 +1,396 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
+RETURNS text AS
+$$
+DECLARE
+  sql TEXT;
+  id INTEGER;
+  rec RECORD;
+  lag INTERVAL;
+  tmp INTEGER;
+  ogc_geom geometry_columns; -- old the_geom record in geometry_columns
+  ogc_merc geometry_columns; -- old the_geom_webmercator record in geometry_columns
+  tabtext TEXT;
+BEGIN
+
+  -- Save current constraints on geometry columns, if any
+  ogc_geom = ('','','','',0,0,'GEOMETRY');
+  ogc_merc = ogc_geom;
+  sql := 'SELECT gc.* FROM geometry_columns gc, pg_class c, pg_namespace n '
+    || 'WHERE c.oid = ' || tabname::oid || ' AND n.oid = c.relnamespace'
+    || ' AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname'
+    || ' AND gc.f_geometry_column IN ( ' || quote_literal('the_geom')
+    || ',' || quote_literal('the_geom_webmercator') || ')';
+  FOR rec IN EXECUTE sql LOOP
+    IF rec.f_geometry_column = 'the_geom' THEN
+      ogc_geom := rec;
+    ELSE
+      ogc_merc := rec;
+    END IF;
+  END LOOP;
+
+  tabtext := Format('%s.%s','public',tabname);
+  RAISE NOTICE 'CARTODBFYING % !!!!', tabtext;
+  PERFORM CDB_CartodbfyTable('public', tabname);
+  tabname := tabtext::regclass;
+
+  sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
+  EXECUTE sql INTO STRICT id;
+  sql := 'SELECT the_geom_webmercator FROM '
+    || tabname::text || ' WHERE cartodb_id = ' || id;
+  EXECUTE sql INTO STRICT rec;
+
+  -- Check the_geom_webmercator trigger
+  IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
+    RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
+  END IF;
+  IF round(st_y(rec.the_geom_webmercator)) != 222684 THEN
+    RAISE EXCEPTION 'the_geom_webmercator Y is % (expecting 222684)', round(st_y(rec.the_geom_webmercator));
+  END IF;
+
+  -- Check CDB_TableMetadata entry
+  sql := 'SELECT * FROM CDB_TableMetadata WHERE tabname = ' || tabname::oid;
+  EXECUTE sql INTO STRICT rec;
+  lag = rec.updated_at - now();
+  IF lag > '1 second' THEN
+    RAISE EXCEPTION 'updated_at in CDB_TableMetadata not set to now() after insert [ valued % ago ]', lag;
+  END IF;
+
+  -- Check geometry_columns entries
+  tmp := 0;
+  FOR rec IN
+    SELECT
+      CASE WHEN gc.f_geometry_column = 'the_geom' THEN 4326
+           ELSE 3857 END as expsrid,
+      CASE WHEN gc.f_geometry_column = 'the_geom' THEN ogc_geom.type
+           ELSE ogc_merc.type END as exptype, gc.*
+    FROM geometry_columns gc, pg_class c, pg_namespace n
+    WHERE c.oid = tabname::oid AND n.oid = c.relnamespace
+          AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname
+          AND gc.f_geometry_column IN ( 'the_geom', 'the_geom_webmercator')
+  LOOP
+    tmp := tmp + 1;
+    -- Check SRID constraint
+    IF rec.srid != rec.expsrid THEN
+      RAISE EXCEPTION 'SRID of % in geometry_columns is %, expected %',
+        rec.f_geometry_column, rec.srid, rec.expsrid;
+    END IF;
+    -- Check TYPE constraint didn't change
+    IF (rec.type != 'GEOMETRY') AND (rec.type != 'POINT') THEN
+      RAISE EXCEPTION 'type of % in geometry_columns is %, expected %',
+        rec.f_geometry_column, rec.type, rec.exptype;
+    END IF;
+    -- check coord_dimension ?
+  END LOOP;
+  IF tmp != 2 THEN
+      RAISE EXCEPTION '% entries found for table % in geometry_columns, expected 2', tmp, tabname;
+  END IF;
+
+  -- Check GiST index
+  sql := 'SELECT a.attname, count(ri.relname) FROM'
+    || ' pg_index i, pg_class c, pg_class ri, pg_attribute a, pg_opclass o'
+    || ' WHERE i.indrelid = c.oid AND ri.oid = i.indexrelid'
+    || ' AND a.attrelid = ri.oid AND o.oid = i.indclass[0]'
+    || ' AND a.attname IN ( ' || quote_literal('the_geom')
+    || ',' || quote_literal('the_geom_webmercator') || ')'
+    || ' AND ri.relnatts = 1 AND o.opcname = '
+    || quote_literal('gist_geometry_ops_2d')
+    || ' AND c.oid = ' || tabname::oid
+    || ' GROUP BY a.attname';
+  RAISE NOTICE 'sql: %', sql;
+  EXECUTE sql;
+  GET DIAGNOSTICS tmp = ROW_COUNT;
+  IF tmp != 2 THEN
+      RAISE EXCEPTION '% gist indices found on the_geom and the_geom_webmercator, expected 2', tmp;
+  END IF;
+
+  -- Check null constraint on cartodb_id, created_at, updated_at
+  SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
+    AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
+      ( 'cartodb_id' )
+    AND NOT a.attnotnull INTO strict tmp;
+  IF tmp > 0 THEN
+      RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
+  END IF;
+
+  -- Cleanup
+  sql := 'DELETE FROM ' || tabname::text || ' WHERE cartodb_id = ' || id;
+  EXECUTE sql;
+
+  RETURN label || ' cartodbfied fine';
+END;
+$$
+LANGUAGE 'plpgsql';
+
+-- check cartodbfytable idempotence
+CREATE TABLE t AS SELECT 1::int as a;
+SELECT CDB_CartodbfyTable('public', 't'); -- should fail
+SELECT CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
+SELECT CDB_CartodbfyTableCheck('t', 'single non-geometrical column');
+DROP TABLE t;
+
+-- table with single non-geometrical column
+CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+DROP TABLE t;
+
+-- table with existing srid-unconstrained (but type-constrained) the_geom
+CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(0,0),4326)::geometry(point) as the_geom;
+SELECT CDB_CartodbfyTableCheck('t', 'srid-unconstrained the_geom');
+DROP TABLE t;
+
+-- table with mixed-srid the_geom values
+CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom
+UNION ALL SELECT ST_SetSRID(ST_MakePoint(0,0),3857);
+SELECT CDB_CartodbfyTableCheck('t', 'mixed-srid the_geom');
+SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
+DROP TABLE t;
+
+-- table with wrong srid-constrained the_geom values
+CREATE TABLE t AS SELECT 'SRID=3857;LINESTRING(222638.981586547 222684.208505545, 111319.490793274 111325.142866385)'::geometry(geometry,3857) as the_geom;
+SELECT CDB_CartodbfyTableCheck('t', 'wrong srid-constrained the_geom');
+SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)),ST_Extent(ST_SnapToGrid(the_geom_webmercator,1)) FROM t;
+DROP TABLE t;
+
+-- table with wrong srid-constrained the_geom_webmercator values (and no the_geom!)
+CREATE TABLE t AS SELECT 'SRID=4326;LINESTRING(1 1,2 2)'::geometry(geometry,4326) as the_geom_webmercator;
+SELECT CDB_CartodbfyTableCheck('t', 'wrong srid-constrained the_geom_webmercator');
+-- expect the_geom to be populated from the_geom_webmercator
+SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
+DROP TABLE t;
+
+-- table with existing triggered the_geom
+CREATE TABLE t AS SELECT 'SRID=4326;LINESTRING(1 1,2 2)'::geometry(geometry) as the_geom;
+CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE UPDATE OF the_geom ON t
+ FOR EACH ROW EXECUTE PROCEDURE _CDB_update_the_geom_webmercator();
+SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
+SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
+DROP TABLE t;
+
+-- table with existing cartodb_id field of type text
+CREATE TABLE t AS SELECT 10::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+select cartodb_id/2 FROM t;
+DROP TABLE t;
+
+-- table with existing cartodb_id field of type text not casting
+CREATE TABLE t AS SELECT 'nan'::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+DROP TABLE t;
+
+-- table with empty cartodb_id field of type text
+CREATE TABLE t AS SELECT null::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'empty text cartodb_id');
+SELECT cartodb_id from t;
+DROP TABLE t;
+
+-- table with existing cartodb_id field of type int4 not sequenced
+CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'unsequenced cartodb_id');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column
+CREATE TABLE t AS SELECT 'SRID=4326;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, no SRS
+CREATE TABLE t AS SELECT 'POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, no srs');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, unusual SRS
+CREATE TABLE t AS SELECT 'SRID=26910;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, srs = 26819');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text unparseable geometry column
+CREATE TABLE t AS SELECT 'SRID=26910;PONT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, unparseable content');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with existing cartodb_id serial primary key
+CREATE TABLE t ( cartodb_id serial primary key );
+SELECT CDB_CartodbfyTableCheck('t', 'cartodb_id serial primary key');
+SELECT c.conname, a.attname FROM pg_constraint c, pg_attribute a
+WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
+AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
+DROP TABLE t;
+
+-- tables can be renamed and there's no index name clashing #123
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+ALTER TABLE original RENAME TO original_renamed;
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+DROP TABLE original_renamed;
+DROP TABLE original;
+
+-- Table always have a default seq value after cartodbfy #138
+CREATE TABLE bug_empty_table_no_seq (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+SELECT CDB_CartodbfyTableCheck('bug_empty_table_no_seq', 'Table always have a default seq value after cartodbfy #138');
+INSERT INTO bug_empty_table_no_seq DEFAULT VALUES;
+DROP TABLE bug_empty_table_no_seq;
+
+-- Existing cartodb_id values are respected
+CREATE table existing_cartodb_id (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+INSERT INTO existing_cartodb_id (cartodb_id, description) VALUES
+       (10, 'a'),
+       (20, 'b'),
+       (30, 'c');
+SELECT CDB_CartodbfyTableCheck('existing_cartodb_id', 'Existing cartodb_id values are respected #138');
+SELECT cartodb_id,the_geom,the_geom_webmercator,description,name from existing_cartodb_id;
+DROP TABLE existing_cartodb_id;
+
+-- Table with both the_geom and wkb_geometry
+CREATE TABLE many_geometry_columns (
+       the_geom geometry,
+       wkb_geometry geometry(MultiPoint,4326),
+       description varchar
+);
+INSERT INTO many_geometry_columns (the_geom, wkb_geometry) VALUES
+       ('0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440', '0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440'),
+       ('0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440', '0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440');
+SELECT CDB_CartodbfyTableCheck('many_geometry_columns', 'Table with both the_geom and wkb_geometry #141');
+SELECT * FROM many_geometry_columns;
+DROP TABLE many_geometry_columns;
+
+-- Many colliding geom columns
+CREATE TABLE many_colliding_columns (
+  the_geom varchar,
+  the_geom_webmercator varchar,
+  my_geom geometry,
+  my_mercgeom geometry(Point, 3857),
+  cartodb_id varchar,
+  my_pk integer primary key
+);
+INSERT INTO many_colliding_columns VALUES (
+ 'foo', 'bar', 'SRID=4326;POINT(0 0)', 'SRID=3857;POINT(0 0)', 'nerf', 1
+);
+SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141');
+DROP TABLE many_colliding_columns;
+
+-- Table with null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+-- Table with non unique cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+-- Table with non unique and null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  cartodb_id integer
+);
+CREATE UNIQUE INDEX "test_cartodb_id_key" ON test (cartodb_id);
+CREATE UNIQUE INDEX "test_cartodb_id_pkey" ON test (cartodb_id);
+ALTER TABLE test ADD CONSTRAINT "test_pkey" PRIMARY KEY USING INDEX test_cartodb_id_pkey;
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with primary key and unique index on it #174');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  name varchar,
+  "first.value" integer,
+  "second.value" integer
+);
+INSERT INTO test VALUES ('one', 1, 2), ('two', 3, 4);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with dots in name columns (cartodb #6114)');
+SELECT name, "first.value" from test;
+DROP TABLE test;
+
+SET client_min_messages TO notice;
+-- _CDB_create_cartodb_id_column with cartodb_id integer already present
+CREATE TABLE test (cartodb_id integer);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+
+-- _CDB_create_cartodb_id_column with cartodb_id text already present
+CREATE TABLE test (cartodb_id text);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+SET client_min_messages TO error;
+
+-- Unique identifier generation can break CDB_CartodbfyTable #305
+BEGIN;
+    DO $$
+    BEGIN
+       FOR i IN 1..150 LOOP
+            EXECUTE 'CREATE TABLE untitled_table();';
+            EXECUTE $query$SELECT CDB_CartodbfyTable('untitled_table');$query$;
+            EXECUTE 'ALTER TABLE untitled_table RENAME TO my_renamed_table_' || i;
+       END LOOP;
+    END;
+    $$;
+ROLLBACK;
+
+-- Long table name could cause possible sequence rename collision #325
+CREATE TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8" ( cartodb_id serial primary key );
+SELECT CDB_CartodbfyTableCheck('wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8'::REGCLASS, 'Long table name could cause sequence collision while renaming #325');
+DROP TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8";
+
+-- TODO: table with existing custom-triggered the_geom
+
+DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);
+DROP FUNCTION _CDB_UserQuotaInBytes();

lib/sql/test/CDB_CartodbfyTableTest_expect

@@ -0,0 +1,157 @@
+SET
+CREATE FUNCTION
+SELECT 1
+ERROR:  Please set user quota before cartodbfying tables.
+0
+single non-geometrical column cartodbfied fine
+DROP TABLE
+SELECT 1
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+DROP TABLE
+SELECT 1
+srid-unconstrained the_geom cartodbfied fine
+DROP TABLE
+SELECT 2
+mixed-srid the_geom cartodbfied fine
+extent|BOX(-1 -1,0 0)
+DROP TABLE
+SELECT 1
+wrong srid-constrained the_geom cartodbfied fine
+extent|BOX(1 1,2 2)|BOX(111319 111325,222639 222684)
+DROP TABLE
+SELECT 1
+wrong srid-constrained the_geom_webmercator cartodbfied fine
+extent|BOX(1 1,2 2)
+DROP TABLE
+SELECT 1
+CREATE TRIGGER
+trigger-protected the_geom cartodbfied fine
+extent|BOX(1 1,2 2)
+DROP TABLE
+SELECT 1
+text cartodb_id cartodbfied fine
+5
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+
+DROP TABLE
+SELECT 1
+unsequenced cartodb_id cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, no srs cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, srs = 26819 cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, unparseable content cartodbfied fine
+1
+DROP TABLE
+CREATE TABLE
+cartodb_id serial primary key cartodbfied fine
+t_pkey|cartodb_id
+DROP TABLE
+CREATE TABLE
+original
+ALTER TABLE
+CREATE TABLE
+original
+DROP TABLE
+DROP TABLE
+CREATE TABLE
+Table always have a default seq value after cartodbfy #138 cartodbfied fine
+INSERT 0 1
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Existing cartodb_id values are respected #138 cartodbfied fine
+10|||a|
+20|||b|
+30|||c|
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with both the_geom and wkb_geometry #141 cartodbfied fine
+1|0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440|0104000020110F00000100000001010000004A9F662B456D5FC11392690DC3F75241|
+2|0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440|0104000020110F00000100000001010000002858E0EC376D5FC1CAE8DB4B95F55241|
+DROP TABLE
+CREATE TABLE
+INSERT 0 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: multiple primary keys for table "many_colliding_columns" are not allowed): ALTER TABLE many_colliding_columns ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+3
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+2
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+2
+DROP TABLE
+CREATE TABLE
+CREATE INDEX
+CREATE INDEX
+ALTER TABLE
+INSERT 0 3
+Table with primary key and unique index on it #174 cartodbfied fine
+1
+2
+3
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with dots in name columns (cartodb #6114) cartodbfied fine
+one|1
+two|3
+DROP TABLE
+SET
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+SET
+BEGIN
+DO
+ROLLBACK
+CREATE TABLE
+Long table name could cause sequence collision while renaming #325 cartodbfied fine
+DROP TABLE
+DROP FUNCTION
+DROP FUNCTION

lib/sql/test/CDB_DateToNumberTest.sql

@@ -0,0 +1,2 @@
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00'::timestamp);
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+05'::timestamp with time zone);

lib/sql/test/CDB_DateToNumberTest_expect

@@ -0,0 +1,2 @@
+915753600
+915735600

lib/sql/test/CDB_DigitSeparatorTest.sql

@@ -0,0 +1,39 @@
+BEGIN;
+CREATE TEMP TABLE "_CDB_DigitSeparatorTest" (
+  none text,
+  only_com_dec text,
+  only_dot_dec text,
+  only_com_tho text,
+  only_dot_tho text,
+  both_com_dec text,
+  both_dot_dec text,
+  "only_com_AMB" text,
+  "only_dot_AMB" text
+);
+COPY "_CDB_DigitSeparatorTest" FROM STDIN;
+123456	123,1235	123.12345	1,234,231	1.234.234	1.234,23	1,234.23	1,123	1.123
+123456	123,12	123.12	231	234	1.121.234,230	3,111,234.230	123,123	123.123
+123456	123,12	123.12	231	234	1.121.234,2	3,111,234.230	123,123	123.123
+\.
+
+SELECT 'none', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'none');
+SELECT 'only_com_dec', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_com_dec');
+SELECT 'only_dot_dec', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_dot_dec');
+SELECT 'only_com_tho', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_com_tho');
+SELECT 'only_dot_tho', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_dot_tho');
+SELECT 'both_com_dec', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'both_com_dec');
+SELECT 'both_dot_dec', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'both_dot_dec');
+SELECT 'only_com_AMB', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_com_AMB');
+SELECT 'only_dot_AMB', * FROM
+       CDB_DigitSeparator('"_CDB_DigitSeparatorTest"'::regclass, 'only_dot_AMB');
+
+DROP TABLE "_CDB_DigitSeparatorTest";
+COMMIT;

lib/sql/test/CDB_DigitSeparatorTest_expect

@@ -0,0 +1,14 @@
+BEGIN
+CREATE TABLE
+COPY 3
+none||
+only_com_dec|.|,
+only_dot_dec|,|.
+only_com_tho|,|.
+only_dot_tho|.|,
+both_com_dec|.|,
+both_dot_dec|,|.
+only_com_AMB||
+only_dot_AMB||
+DROP TABLE
+COMMIT

lib/sql/test/CDB_DistTypeTest.sql

@@ -0,0 +1,4 @@
+WITH data AS (
+    SELECT pow(x,3)::numeric x FROM generate_series(-100,100) x
+    ) 
+SELECT CDB_DistType(array_agg(x)) FROM data

lib/sql/test/CDB_DistTypeTest_expect

@@ -0,0 +1 @@
+A

lib/sql/test/CDB_DistinctMeasureTest.sql

@@ -0,0 +1,20 @@
+-- a - j add up to 89%, k-m add up to 11%
+WITH a As (
+	SELECT (
+		repeat('a',12) || 
+		repeat('b',11) || 
+		repeat('c',11) || 
+		repeat('d',10) || 
+		repeat('e',10) || 
+		repeat('f',9) || 
+		repeat('g',8) || 
+		repeat('h',7) || 
+		repeat('i',6) || 
+		repeat('j',5) || 
+		repeat('k',4) || 
+		repeat('l',4) || 
+		repeat('m',3)
+		)::text AS x 
+	) 
+
+SELECT CDB_DistinctMeasure(string_to_array(x,null),0.90) from a

lib/sql/test/CDB_DistinctMeasureTest_expect

@@ -0,0 +1 @@
+0

lib/sql/test/CDB_EqualIntervalBinsTest.sql

@@ -0,0 +1,11 @@
+WITH data AS (
+    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x 
+        WHERE x % 5 != 0 AND x % 7 != 0
+    ) 
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data_nulls;

lib/sql/test/CDB_EqualIntervalBinsTest_expect

@@ -0,0 +1,14 @@
+43.5714286
+86.1428571
+128.7142857
+171.2857143
+213.8571429
+256.4285714
+299.0000000
+15.0000000
+29.0000000
+43.0000000
+57.0000000
+71.0000000
+85.0000000
+99.0000000

lib/sql/test/CDB_EstimateRowCountTest.sql

@@ -0,0 +1,10 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+CREATE TABLE tmptab1(id INT);
+INSERT INTO tmptab1(id) VALUES (1), (2), (3);
+CREATE TABLE tmptab2(id INT, value NUMERIC);
+INSERT INTO tmptab2(id, value) VALUES (1, 10.0), (2, 20.0);
+SELECT CDB_EstimateRowCount('SELECT SUM(value) FROM tmptab1 INNER JOIN tmptab2 ON (tmptab1.id = tmptab2.id);') AS row_count;
+SELECT CDB_EstimateRowCount('UPDATE tmptab2 SET value = 30 WHERE id=2;') AS row_count;
+DROP TABLE tmptab2;
+DROP TABLE tmptab1;

lib/sql/test/CDB_EstimateRowCountTest_expect

@@ -0,0 +1,9 @@
+SET
+CREATE TABLE
+INSERT 0 3
+CREATE TABLE
+INSERT 0 2
+1
+1
+DROP TABLE
+DROP TABLE

lib/sql/test/CDB_GhostTables.sql

@@ -0,0 +1,52 @@
+-- Create user and enable Ghost tables trigger
+\set QUIET on
+SET client_min_messages TO error;
+SELECT CDB_EnableGhostTablesTrigger();
+CREATE ROLE "fulano" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "fulano";
+GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano";
+SELECT cartodb.CDB_Conf_SetConf('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+DELETE FROM cdb_conf WHERE key = 'invalidation_service';
+SET SESSION AUTHORIZATION "fulano";
+SET client_min_messages TO notice;
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called (configuration not found)
+
+-- Add TIS configuration
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT cartodb.CDB_Conf_SetConf('invalidation_service', '{"host": "fake-tis-host", "port": 3142}');
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called
+
+BEGIN;
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0
+CREATE TABLE tmp(id INT);
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1
+END; -- _CDB_LinkGhostTables called
+
+-- Disable Ghost tables trigger
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableGhostTablesTrigger();
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists
+DROP TABLE tmp; -- _CDB_LinkGhostTables not called
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano";
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano";
+REVOKE ALL ON SCHEMA cartodb FROM "fulano";
+DROP ROLE "fulano";
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service';
+\set QUIET off

lib/sql/test/CDB_GhostTables_expect

@@ -0,0 +1,20 @@
+
+
+WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+BEGIN
+cdb_ddl_execution
+0
+CREATE TABLE
+1
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+COMMIT
+
+
+DROP TABLE

lib/sql/test/CDB_GreatCircle.sql

@@ -0,0 +1,2 @@
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)));
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000));

lib/sql/test/CDB_GreatCircle_expect

@@ -0,0 +1,2 @@
+LINESTRING(4.259 55.858,5.53349240387128 56.0006659105918,6.81698919498694 56.130094578525,8.10870381314147 56.2461317260662,9.40781156033233 56.3486370295466,10.7134527044527 56.4374849223869,12.0247359780093 56.5125653297878,13.3407424424749 56.573784325367,14.660529681225 56.6210647008095,15.9831362768927 56.654346440595,17.307586522649 56.67358709506,18.6328953115992 56.6787620464102,19.9580731443722 56.6698646638042,21.282131192215 56.6469063452276,22.6040863516019 56.6099164455407,23.922966226566 56.5589420917603,25.2378139766594 56.4940478882858,26.5476929715805 56.4153155163602,27.8516911979552 56.3228432335229,29.1489253693643 56.2167452801302,30.4385446972665 56.0971512011646,31.7197342877491 55.9642050924945,32.9917181368037 55.8180647814723,34.2537617048216 55.6589009522625,35.5051740589896 55.4868962265697,36.7453095800251 55.3022442104976,37.9735692370026 55.1051485181267,39.1894014407465 54.8958217820713,40.3923024922398 54.6744846607816,41.5818166476476 54.4413648517294,42.757535825811 54.1966961188706,43.919098987406 53.9407173419567,45.0661912174019 53.673671594382,46.198542544017 53.3958052553427,47.3159265281308 53.1073671611612,48.4181586571351 52.8086077997244,49.5050945765883 52.499778551104,50.5766281918714 52.1811309766006,51.6326896704254 51.8529161576737,52.673243373185 51.5153840855177,53.6982857415906 51.1687831014009,54.7078431641625 50.8133593873253,55.7019698441171 50.4493565060761,56.6807456869812 50.0770149893128,57.6442742246566 49.6965719720156,58.5926805899637 49.3082608713202,59.5261095533829 48.9123111075629,60.4447236315382 48.5089478652008,61.3487012749643 48.0983918911668,62.2382351408597 47.6808593281578,63.1135304548766 47.2565615803358,63.9748034645285 46.8257052089336,64.822279985501 46.3884918552974,65.6561940410346 45.9451181889661,66.476786593589 45.4957758784676,67.284304367196 45.0406515826125,68.0789987582454 44.5799269601738,68.8611248319107 44.1137786959568,69.6309404010034 43.6423785413868,70.388705183725 43.1658933678633,71.1346800365587 42.6844852312539,71.8691262583921 42.1983114460249,72.5923049618788 41.7075246676227,73.3044765080245 41.2122729818388,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.89507305967085 55.930977446384,5.53349240387128 56.0006659105918,6.17416348361598 56.0670448594645,6.81698919498694 56.130094578525,7.46186995983655 56.1897961993418,8.10870381314147 56.2461317260662,8.75738649688733 56.2990840610623,9.40781156033233 56.3486370295466,10.0598704664666 56.3947754031591,10.7134527044527 56.4374849223869,11.3684459078018 56.4767523177655,12.0247359780093 56.5125653297878,12.6822072133468 56.5449127274491,13.3407424424749 56.573784325367,14.0002231625192 56.5991709994144,14.660529681225 56.6210647008095,15.3215412627822 56.6394584686143,15.9831362768927 56.654346440595,16.6451923506331 56.6657238624055,17.307586522649 56.67358709506,17.9701953992046 56.677933620668,18.6328953115992 56.6787620464102,19.2955624744544 56.6760721067401,19.9580731443722 56.6698646638042,20.6203037784591 56.6601417060788,21.282131192215 56.6469063452276,21.943432716288 56.6301628111935,22.6040863516019 56.6099164455407,23.2639709223762 56.5861736930735,23.922966226566 56.5589420917603,24.5809531832687 56.5282302610022,25.2378139766594 56.4940478882858,25.8934321960358 56.4564057142701,26.5476929715805 56.4153155163602,27.2004831054654 56.3707900908252,27.8516911979552 56.3228432335229,28.5012077681911 56.2714897192993,29.1489253693643 56.2167452801302,29.7947386980206 56.1586265820819,30.4385446972665 56.0971512011646,31.0802426536785 56.0323375981587,31.7197342877491 55.9642050924945,32.3569238377352 55.8927738352675,32.9917181368037 55.8180647814723,33.6240266834038 55.74009966154,34.2537617048216 55.6589009522625,34.8808382139074 55.5744918471876,35.5051740589896 55.4868962265697,36.1266899670207 55.3961386269571,36.7453095800251 55.3022442104976,37.3609594849451 55.2052387340427,37.9735692370026 55.1051485181267,38.5830713767178 55.0020004158976,39.1894014407465 54.8958217820713,39.7924979667135 54.7866404419798,40.3923024922398 54.6744846607816,40.9887595483734 54.5593831128969,41.5818166476476 54.4413648517294,42.1714242670021 54.3204592797319,42.757535825811 54.1966961188706,43.3401076592708 54.0701053815371,43.919098987406 53.9407173419567,44.4944718799548 53.8085625081347,45.0661912174019 53.673671594382,45.6342246484243 53.5360754944551,46.198542544017 53.3958052553427,46.7591179485663 53.2528920517295,47.3159265281308 53.1073671611612,47.8689465161932 52.9592619399335,48.4181586571351 52.8086077997244,48.9635461476859 52.6554361849853,49.5050945765883 52.499778551104,50.0427918627159 52.3416663433486,50.5766281918714 52.1811309766006,51.1065959524853 52.0182038158815,51.6326896704254 51.8529161576737,52.1549059431199 51.6852992120372,52.673243373185 51.5153840855177,53.1877025017413 51.3432017648431,53.6982857415906 51.1687831014009,54.2049973104167 50.9921587964881,54.7078431641625 50.8133593873253,55.2068309307272 50.6324152338211,55.7019698441171 50.4493565060761,56.1932706791714 50.2642131726125,56.6807456869812 50.0770149893128,57.1644085311015 49.8877914890534,57.6442742246566 49.6965719720156,58.1203590684218 49.5033854966561,58.5926805899637 49.3082608713202,59.0612574839055 49.1112266464775,59.5261095533829 48.9123111075629,59.9872576527434 48.7115422684016,60.4447236315382 48.5089478652008,60.8985302798459 48.3045553510865,61.3487012749643 48.0983918911668,61.7952611294973 47.8904843581013,62.2382351408597 47.6808593281578,62.6776493422177 47.4695430777354,63.1135304548766 47.2565615803358,63.5459058421237 47.0419405039633,63.9748034645285 46.8257052089336,64.4002518367009 46.6078807460742,64.822279985501 46.3884918552974,65.2409174096934 46.1675629645276,65.6561940410346 45.9451181889661,66.0681402067793 45.7211813306754,66.476786593589 45.4957758784676,66.8821642128236 45.2689250080781,67.284304367196 45.0406515826125,67.6832386187654 44.8109781532476,68.0789987582454 44.5799269601738,68.4716167756018 44.3475199337644,68.8611248319107 44.1137786959568,69.2475552324516 43.878724561833,69.6309404010034 43.6423785413868,70.0113128553159 43.4047613414637,70.388705183725 43.1658933678633,70.7631500228809 42.925794727592,71.1346800365587 42.6844852312539,71.5033278955199 42.4419843955718,71.8691262583921 42.1983114460249,72.2321077535378 41.953485319597,72.5923049618788 41.7075246676227,72.9497504006463 41.4604478587259,73.3044765080245 41.2122729818388,73.6565156286596 40.963017849297,74.0059 40.7127)

lib/sql/test/CDB_HeadsTailsBinsTest.sql

@@ -0,0 +1,11 @@
+WITH data AS (
+    SELECT array_agg(x::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT round(unnest(CDB_HeadsTailsBins(s, 7)),2) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT round(unnest(CDB_HeadsTailsBins(s, 7)),2) FROM data_nulls;

lib/sql/test/CDB_HeadsTailsBinsTest_expect

@@ -0,0 +1,13 @@
+49.56
+74.44
+87.50
+93.50
+96.50
+98.00
+99.00
+49.76
+74.65
+88.50
+94.50
+98.00
+99.00

lib/sql/test/CDB_HelperTest.sql

@@ -0,0 +1,138 @@
+-- Test unique identifier creation with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', NULL);
+
+-- Test unique identifier creation with prefix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'relname', NULL);
+
+-- Test unique identifier creation with suffix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', '_suffix');
+
+-- Test unique identifier creation with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test unique identifier creation with prefix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargola (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargola;
+
+-- Test unique identifier creation with suffix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE largolargolargolargolargolargolargolargolargolargola_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+DROP TABLE largolargolargolargolargolargolargolargolargolargola_suffix;
+
+-- Test unique identifier creation with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', NULL);
+
+-- Test unique identifier creation with prefix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piraña', NULL);
+
+-- Test unique identifier creation with suffix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', '_suffix');
+
+-- Test unique identifier creation with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test unique identifier creation with prefix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+DROP TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi;
+
+-- Test unique identifier creation with suffix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargola text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length normal colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (largolargolargolargolargolargolargolargolargolargola_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length UTF8 colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test _CDB_Octet_Truncate simple case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 5);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 6);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 7);
+
+-- Test _CDB_Table_Exists
+CREATE TABLE public.this_table_exists();
+SELECT cartodb._CDB_Table_Exists('this_table_does_not_exist');
+SELECT cartodb._CDB_Table_Exists('this_schema_does_not_exist.this_table_does_not_exist');
+SELECT cartodb._CDB_Table_Exists('this_table_exists');
+SELECT cartodb._CDB_Table_Exists('public.this_table_exists');
+SELECT cartodb._CDB_Table_Exists('raster_overviews'); -- view created by postgis
+SELECT cartodb._CDB_Table_Exists('public.raster_overviews');
+DROP TABLE public.this_table_exists

lib/sql/test/CDB_HelperTest_expect

@@ -0,0 +1,67 @@
+relname
+prefix_relname
+relname_suffix
+largolargolargolargolargolargolargolargolargolargolargolarg
+prefix_largolargolargolargolargolargolargolargolargolargola
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargola0
+DROP TABLE
+largolargolargolargolargolargolargolargolargolargola_suffix
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargola_suffix0
+DROP TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaáci
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi0
+DROP TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix0
+DROP TABLE
+CREATE TABLE
+colname
+prefix_colname
+colname_suffix
+largolargolargolargolargolargolargolargolargolargolargolarg
+prefix_largolargolargolargolargolargolargolargolargolargola
+DROP TABLE
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargola0
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargola_suffix
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargola_suffix0
+DROP TABLE
+CREATE TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaáci
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+DROP TABLE
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi0
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix0
+DROP TABLE
+pira
+pirañ
+piraña
+CREATE TABLE
+f
+f
+t
+t
+f
+f
+DROP TABLE

lib/sql/test/CDB_HexagonTest.sql

@@ -0,0 +1,56 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
+-- Check correctness of an hexagons grid
+--
+-- Cells must have no overlaps and have a number of
+-- intersections with other cells between 2 and 6
+--
+
+WITH
+ params AS ( SELECT ST_MakeEnvelope(10, 10, 20, 20) as env, 2 as radius ),
+ grid AS ( SELECT CDB_HexagonGrid(env, radius) AS cell from params),
+ igrid AS ( SELECT row_number() over (), cell from grid )
+
+ SELECT count(row_number) as r1, sum(st_npoints(cell)) as r2, 'count / npoints' as err
+ FROM igrid g1
+
+UNION ALL
+
+ SELECT g1.row_number as r1, g2.row_number as r2, 'overlap' as err
+ FROM igrid g1, igrid g2
+ WHERE g2.row_number > g1.row_number AND
+       ST_Overlaps(g1.cell, g2.cell)
+
+UNION ALL
+
+ SELECT g1.row_number, count(g2.row_number) as r2, 'n intersections' as err
+ FROM igrid g1, igrid g2
+ WHERE g1.row_number != g2.row_number AND
+       ST_Intersects(g1.cell, g2.cell)
+       GROUP BY g1.row_number
+       HAVING count(g2.row_number) > 6 OR count(g2.row_number) < 2
+
+UNION ALL
+
+ SELECT g1.row_number, null::integer, 'centroid out of extent' as err
+ FROM igrid g1, params
+ WHERE NOT ST_Intersects(ST_Centroid(g1.cell), params.env)
+
+	;
+
+
+-- Check robustness of HexagonGrid generation
+
+WITH
+ params AS ( SELECT
+ ST_MakeEnvelope(-20037508.5,20037507,-20037508,20037507.5) as env,
+ 0.002 as radius ),
+ grid AS ( SELECT CDB_HexagonGrid(env, radius) AS cell from params)
+SELECT '#160', count(cell) > 23000 from grid;
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1);

lib/sql/test/CDB_HexagonTest_expect

@@ -0,0 +1,5 @@
+SET
+9|63|count / npoints
+#160|t
+3886
+ERROR:  The requested grid is too big to be rendered

lib/sql/test/CDB_JenksBinsTest.sql

@@ -0,0 +1,36 @@
+WITH data AS (
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5)) FROM data;
+
+
+WITH data_nulls AS (
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 null, null,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 null, null,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5)) FROM data_nulls;
+
+
+WITH data_inverse AS (
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5, 0, true)) FROM data_inverse;
+
+
+WITH data_small AS (
+    SELECT Array[0.99, 1.0, 10.01, 10.01, 10.01, 10.01]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 4)) FROM data_small;

lib/sql/test/CDB_JenksBinsTest_expect

@@ -0,0 +1,18 @@
+1.01
+5.01
+10.01
+15.01
+20.1
+1.01
+5.01
+10.01
+15.01
+20.1
+0.99
+4.99
+10.01
+14.99
+19.9
+0.99
+1.0
+10.01

lib/sql/test/CDB_MathTest.sql

@@ -0,0 +1,4 @@
+
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1]) a;

lib/sql/test/CDB_MathTest_expect

@@ -0,0 +1,3 @@
+2
+1
+1

lib/sql/test/CDB_OverviewsTest.sql

@@ -0,0 +1,44 @@
+SET client_min_messages TO error;
+\set VERBOSITY default
+
+\i test/overviews/fixtures.sql
+vacuum ANALYZE; -- Make sure there are metrics for ST_EstimatedExtent
+
+SELECT _CDB_Aggregable_Attributes_Expression('base_bare_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
+
+SELECT CDB_CreateOverviews('base_bare_t'::regclass);
+SELECT count(*) FROM _vovw_2_base_bare_t;
+
+
+SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
+
+SELECT CDB_CreateOverviews('base_t'::regclass);
+SELECT count(*) FROM _vovw_2_base_t;
+
+SELECT CDB_CreateOverviews('polyg_t'::regclass);
+
+SELECT CDB_CreateOverviews('column_types_t'::regclass);
+
+SELECT CDB_Overviews('base_t'::regclass);
+SELECT CDB_Overviews('"public"."base_t"'::regclass);
+SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
+SELECT CDB_Overviews('polyg_t'::regclass);
+SELECT CDB_Overviews('column_types_t'::regclass);
+
+SELECT CDB_DropOverviews('column_types_t'::regclass);
+SELECT CDB_DropOverviews('base_bare_t'::regclass);
+SELECT CDB_DropOverviews('base_t'::regclass);
+SELECT count(*) FROM _vovw_2_base_t;
+
+SELECT CDB_CreateOverviewsWithToleranceInPixels('base_t'::regclass, 7.5);
+SELECT count(*) FROM _vovw_2_base_t;
+SELECT CDB_DropOverviews('base_t'::regclass);
+
+DROP TABLE column_types_t;
+DROP TABLE base_bare_t;
+DROP TABLE base_t;
+DROP TABLE polyg_t;

lib/sql/test/CDB_OverviewsTest_expect

@@ -0,0 +1,49 @@
+SET
+CREATE TABLE
+INSERT 0 1114
+CREATE TABLE
+INSERT 0 1114
+CREATE TABLE
+INSERT 0 5
+SELECT 1114
+VACUUM
+
+
+
+{_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+126
+number,int_number,name,start
+SUM(number*1)/count(*)::double precision AS number,SUM(int_number*1)/count(*)::integer AS int_number,CASE WHEN count(distinct name) = 1 THEN MIN(name) WHEN count(*) < 5 THEN string_agg(distinct name,' / ') ELSE '*' END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
+SUM(tab.number*1)/count(*)::double precision AS number,SUM(tab.int_number*1)/count(*)::integer AS int_number,CASE WHEN count(distinct tab.name) = 1 THEN MIN(tab.name) WHEN count(*) < 5 THEN string_agg(distinct tab.name,' / ') ELSE '*' END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
+{_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+126
+
+{_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_bare_t,0,_vovw_0_base_bare_t)
+(base_bare_t,1,_vovw_1_base_bare_t)
+(base_bare_t,2,_vovw_2_base_bare_t)
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(column_types_t,0,_vovw_0_column_types_t)
+(column_types_t,1,_vovw_1_column_types_t)
+(column_types_t,2,_vovw_2_column_types_t)
+
+
+
+ERROR:  relation "_vovw_2_base_t" does not exist
+LINE 1: SELECT count(*) FROM _vovw_2_base_t;
+                             ^
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+38
+
+DROP TABLE
+DROP TABLE
+DROP TABLE
+DROP TABLE

lib/sql/test/CDB_QuantileBinsTest.sql

@@ -0,0 +1,17 @@
+WITH data AS (
+    SELECT array_agg(x::numeric) AS s
+    FROM generate_series(0, 99) AS x
+    ) 
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(x::numeric) AS s
+      FROM (
+        SELECT x FROM generate_series(0, 99) AS x
+        UNION ALL
+        SELECT null AS x FROM generate_series(1, 10) AS x
+        ) _wrap
+    )
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data_nulls;

lib/sql/test/CDB_QuantileBinsTest_expect

@@ -0,0 +1,20 @@
+9
+19
+29
+39
+49
+59
+69
+79
+89
+99
+9
+19
+29
+39
+49
+59
+69
+79
+89
+99

lib/sql/test/CDB_QueryStatementsTest.sql

@@ -0,0 +1,66 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+WITH q AS ( SELECT CDB_QueryStatements('
+SELECT * FROM geometry_columns;
+') as statement )
+SELECT '1', row_number() over (), statement FROM q;
+
+WITH q AS ( SELECT CDB_QueryStatements('
+SELECT * FROM geometry_columns
+') as statement )
+SELECT '2', row_number() over (), statement FROM q;
+
+WITH q AS ( SELECT CDB_QueryStatements('
+;;;SELECT * FROM geometry_columns
+') as statement )
+SELECT '3', row_number() over (), statement FROM q;
+
+WITH q AS ( SELECT CDB_QueryStatements($the_param$
+CREATE table "my'tab;le" ("$" int);
+SELECT '1','$$', '$hello$', "$" FROM "my'tab;le";
+CREATE function "hi'there" ("'" text default '$') returns void as $h$ declare a int; b text; begin b='hi'; return; end; $h$ language 'plpgsql';
+SELECT 5;
+$the_param$) as statement )
+SELECT '4', row_number() over (), statement FROM q;
+
+WITH q AS ( SELECT CDB_QueryStatements($the_param$
+INSER INTO "my''""t" values ('''','""'';;');
+SELECT $qu;oted$ hi $qu;oted$;
+$the_param$) as statement )
+SELECT '5', row_number() over (), statement FROM q;
+
+WITH q AS ( SELECT CDB_QueryStatements($the_param$
+SELECT
+1 ; SELECT
+2
+$the_param$) as statement )
+SELECT '6', row_number() over (), statement FROM q;
+
+-- This is an insane input, illegal sql
+-- we are really only testing that it does not
+-- take forever to process..
+-- The actual result is not correct, so if the function
+-- ever gets fixed check if it's better
+WITH q AS ( SELECT CDB_QueryStatements($the_param$
+
+
+
+
+
+
+
+
+    /a
+    $b$
+    $c$d
+    ;
+$the_param$) as statement )
+SELECT '7', row_number() over (), statement FROM q;
+
+
+WITH q AS ( SELECT CDB_QueryStatements($the_param$
+SELECT $quoted$ hi
+$quoted$;
+$the_param$) as statement )
+SELECT '8', row_number() over (), statement FROM q;

lib/sql/test/CDB_QueryStatementsTest_expect

@@ -0,0 +1,20 @@
+SET
+1|1|SELECT * FROM geometry_columns
+2|1|SELECT * FROM geometry_columns
+3|1|SELECT * FROM geometry_columns
+4|1|CREATE table "my'tab;le" ("$" int)
+4|2|SELECT '1','$$', '$hello$', "$" FROM "my'tab;le"
+4|3|CREATE function "hi'there" ("'" text default '$') returns void as $h$ declare a int; b text; begin b='hi'; return; end; $h$ language 'plpgsql'
+4|4|SELECT 5
+5|1|INSER INTO "my''""t" values ('''','""'';;')
+5|2|SELECT $qu;oted$ hi $qu;oted$
+6|1|SELECT
+1
+6|2|SELECT
+2
+7|1|/a
+7|2|b
+7|3|c
+7|4|d
+8|1|SELECT $quoted$ hi
+$quoted$

lib/sql/test/CDB_QueryTablesTest.sql

@@ -0,0 +1,41 @@
+SET client_min_messages TO warning;
+\set VERBOSITY terse
+
+WITH inp AS ( select 'SELECT * FROM geometry_columns'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select $quote$CREATE table "my'tab;le" as select 1$quote$::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'SELECT a.oid, b.oid FROM pg_class a, pg_class b'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'SELECT 1 as col1; select 2 as col2'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'select 1 from nonexistant'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'begin; select * from pg_class; commit;'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'create table test (a int); insert into test values (1); select * from test;'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+WITH inp AS ( select 'WITH a AS (select * from pg_class) select * from a'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+
+CREATE SCHEMA sc;
+create table sc.test (a int);
+insert into sc.test values (1);
+WITH inp AS ( select 'select * from sc.test'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;
+DROP TABLE sc.test;
+DROP SCHEMA sc;
+
+WITH inp AS ( select 'SELECT
+* FROM geometry_columns'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;

lib/sql/test/CDB_QueryTablesTest_expect

@@ -0,0 +1,20 @@
+SET
+SELECT * FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_constraint,pg_catalog.pg_namespace,pg_catalog.pg_type}
+SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)|{pg_catalog.pg_attribute,pg_catalog.pg_class}
+CREATE table "my'tab;le" as select 1|{}
+SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
+SELECT 1 as col1; select 2 as col2|{}
+WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
+ERROR:  relation "nonexistant" does not exist
+begin; select * from pg_class; commit;|{pg_catalog.pg_class}
+WARNING:  CDB_QueryTables cannot explain query: insert into test values (1) (42P01: relation "test" does not exist)
+ERROR:  relation "test" does not exist
+WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
+CREATE SCHEMA
+CREATE TABLE
+INSERT 0 1
+select * from sc.test|{sc.test}
+DROP TABLE
+DROP SCHEMA
+SELECT
+* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_constraint,pg_catalog.pg_namespace,pg_catalog.pg_type}

lib/sql/test/CDB_QuotaTest.sql

@@ -0,0 +1,47 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
+-- See the dice
+SELECT setseed(0.5);
+
+CREATE TABLE big(a int);
+-- Try the legacy interface
+-- See https://github.com/CartoDB/cartodb-postgresql/issues/13
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+INSERT INTO big VALUES (1); -- allowed, check runs before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+SELECT CDB_SetUserQuotaInBytes(0);
+SELECT CDB_CartodbfyTable('big');
+INSERT INTO big SELECT generate_series(2049,4096);
+INSERT INTO big SELECT generate_series(4097,6144);
+INSERT INTO big SELECT generate_series(6145,8192);
+-- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
+-- END Test for #108
+SELECT setseed(0.9);
+SELECT CDB_SetUserQuotaInBytes(2);
+INSERT INTO big VALUES (8193);
+SELECT CDB_SetUserQuotaInBytes(0);
+INSERT INTO big VALUES (8194);
+DROP TABLE big;
+
+
+--analysis tables should be excluded from quota:
+CREATE TABLE big(a int);
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+SELECT CDB_SetUserQuotaInBytes(1);
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int);
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5);
+INSERT INTO big VALUES (1); -- allowed, check runs before
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4;
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+DROP TABLE big;
+SELECT CDB_SetUserQuotaInBytes(0);
+
+
+set client_min_messages to NOTICE;
+DROP FUNCTION _CDB_UserQuotaInBytes();

lib/sql/test/CDB_QuotaTest_expect

@@ -0,0 +1,32 @@
+SET
+
+CREATE TABLE
+CREATE TRIGGER
+INSERT 0 1
+ERROR:  Quota exceeded by 3.9990234375KB
+0
+big
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+454656
+909312
+0
+
+2
+ERROR:  Quota exceeded by 443.998046875KB
+0
+INSERT 0 1
+DROP TABLE
+CREATE TABLE
+CREATE TRIGGER
+1
+CREATE TABLE
+INSERT 0 5
+INSERT 0 1
+DROP TABLE
+ERROR:  Quota exceeded by 3.9990234375KB
+DROP TABLE
+0
+SET
+DROP FUNCTION

lib/sql/test/CDB_RectangleTest.sql

@@ -0,0 +1,8 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10, 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1, 1);

lib/sql/test/CDB_RectangleTest_expect

@@ -0,0 +1,3 @@
+SET
+10000
+ERROR:  The requested grid is too big to be rendered

lib/sql/test/CDB_StatsTest.sql

@@ -0,0 +1,13 @@
+-- continuous uniform distribution has kurtosis = -6/5, skewness = 0.0
+-- http://mathworld.wolfram.com/UniformDistribution.html
+set client_min_messages to ERROR;
+
+WITH dist AS (
+  SELECT generate_series(0,10000)::numeric / 10000.0 i
+)
+SELECT
+  abs(CDB_Kurtosis(array_agg(i)) + 1.2) < 1e-3 AS kurtosis,
+  abs(CDB_Skewness(array_agg(i))) < 1e-3 AS skewness
+FROM dist;
+
+set client_min_messages to NOTICE;

lib/sql/test/CDB_StatsTest_expect

@@ -0,0 +1,3 @@
+SET
+t|t
+SET

lib/sql/test/CDB_SyncTableTest.sql

@@ -0,0 +1,94 @@
+-- Setup: create and populate a table to test the syncs
+\set QUIET on
+BEGIN;
+SET client_min_messages TO error;
+CREATE TABLE test_sync_source (
+  cartodb_id bigint,
+  lat double precision,
+  lon double precision,
+  name text
+);
+INSERT INTO test_sync_source VALUES
+  (1, 1.0, 1.0, 'foo'),
+  (2, 2.0, 2.0, 'bar'),
+  (3, 3.0, 3.0, 'patata'),
+  (4, 4.0, 4.0, 'melon');
+SET client_min_messages TO notice;
+\set QUIET off
+
+
+\echo 'First table sync: it should be simply just copied to the destination'
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Next table sync: there shall be no changes'
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Remove a row from the source and check it is deleted from the dest table'
+DELETE FROM test_sync_source WHERE cartodb_id = 3;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Insert a new row and check that it is inserted in the dest table'
+INSERT INTO test_sync_source VALUES (5, 5.0, 5.0, 'sandia');
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Modify row and check that it is modified in the dest table'
+UPDATE test_sync_source SET name = 'cantaloupe' WHERE cartodb_id = 4;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Sanity check: the end result is the same source table'
+SELECT * FROM test_sync_source ORDER BY cartodb_id;
+SELECT * FROM test_sync_dest ORDER BY cartodb_id;
+
+
+\echo 'It shall exclude geom columns if instructed to do so'
+\set QUIET on
+SET client_min_messages TO error;
+SELECT cartodb.CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
+SELECT cartodb.CDB_CartodbfyTable('test_sync_source');
+SELECT cartodb.CDB_CartodbfyTable('test_sync_dest');
+UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecoding"
+\set QUIET off
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}');
+SELECT * FROM test_sync_source ORDER BY cartodb_id;
+SELECT * FROM test_sync_dest ORDER BY cartodb_id;
+
+\echo 'It will work with schemas that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+CREATE SCHEMA "sch-ema";
+CREATE TABLE "test_sync_source2" AS SELECT * FROM test_sync_source;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test_sync_source2', 'sch-ema', 'test_sync_dest');
+INSERT INTO test_sync_source2(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM test_sync_source2 WHERE cartodb_id = 4;
+UPDATE test_sync_source2 SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source2', 'sch-ema', 'test_sync_dest');
+
+\echo 'It will work with table names that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+CREATE TABLE "test-sync-source" AS SELECT * FROM test_sync_source;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test-sync-source', 'public', 'test-sync-dest');
+INSERT INTO "test-sync-source"(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM "test-sync-source" WHERE cartodb_id = 4;
+UPDATE "test-sync-source" SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test-sync-source', 'public', 'test-sync-dest');
+
+\echo 'It will work with column names that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+ALTER TABLE test_sync_source ADD COLUMN "a-column" int;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest2');
+INSERT INTO test_sync_source(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM test_sync_source WHERE cartodb_id = 4;
+UPDATE test_sync_source SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest2');
+
+-- Cleanup
+ROLLBACK;

lib/sql/test/CDB_SyncTableTest_expect

@@ -0,0 +1,94 @@
+First table sync: it should be simply just copied to the destination
+NOTICE:  INSERTED 4 row(s)
+
+Next table sync: there shall be no changes
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Remove a row from the source and check it is deleted from the dest table
+DELETE 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Insert a new row and check that it is inserted in the dest table
+INSERT 0 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Modify row and check that it is modified in the dest table
+UPDATE 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+Sanity check: the end result is the same source table
+1|1|1|foo
+2|2|2|bar
+4|4|4|cantaloupe
+5|5|5|sandia
+1|1|1|foo
+2|2|2|bar
+4|4|4|cantaloupe
+5|5|5|sandia
+It shall exclude geom columns if instructed to do so
+0
+test_sync_source
+test_sync_dest
+SET
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  DELETED 0 row(s)
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  MODIFIED 0 row(s)
+
+1|||1|1|foo
+2|||2|2|bar
+4|||4|4|cantaloupe
+5|||5|5|sandia
+1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo
+2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar
+4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe
+5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia
+It will work with schemas that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+It will work with table names that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test-sync-dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+It will work with column names that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test_sync_dest2" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+ROLLBACK

lib/sql/test/CDB_TransformToWebmercatorTest.sql

@@ -0,0 +1,28 @@
+BEGIN;
+SET client_min_messages TO error;
+
+-- Run psql with -tA switches and expect
+-- CDB_TransformToWebmercatorTest_expect
+select '1', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;LINESTRING(90 90, 0 80)'), 1));
+select '2', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;LINESTRING(90 90, 0 90)'), 1));
+select '3', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;POINT(0 90)'), 1));
+select '4', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTIPOINT(10 3, 0 90, 0 4)'), 1));
+select '5', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTIPOINT(10 3)'), 1));
+select '6', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTILINESTRING((90 90, 0 90),(0 4, -4 5))'), 1));
+select '7', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;POINT(5 3)'), 1));
+-- See https://github.com/Vizzuality/cartodb/issues/901
+select '8', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;POLYGON((100 0, -100 -100, 100 -100, -100 0, 100 00))'), 1));
+-- See https://github.com/Vizzuality/cartodb/issues/931
+select '9', CDB_TransformToWebmercator(
+ '0106000020E61000000100000001030000000100000007000000010000000000F87F9CDFD01E32095341010000000000F87F193B6F0A30095341010000000000F87FA10FBF4C1D095341010000000000F87F38E258111C095341010000000000F87F5196BAFF17095341010000000000F87F4F0550911B095341010000000000F87F9CDFD01E32095341'::geometry);
+-- Already in webmercator, doun't touch, even if out of valid bounds
+select '10', ST_AsEWKT(CDB_TransformToWebmercator('SRID=3857;POINT(-20037510 -30240972)'::geometry));
+END;

lib/sql/test/CDB_TransformToWebmercatorTest_expect

@@ -0,0 +1,13 @@
+BEGIN
+SET
+1|SRID=3857;LINESTRING(9016879 30240972,0 15538711)
+2|
+3|
+4|SRID=3857;MULTIPOINT(0 445640,1113195 334111)
+5|SRID=3857;MULTIPOINT(1113195 334111)
+6|SRID=3857;MULTILINESTRING((0 445640,-445278 557305))
+7|SRID=3857;POINT(556597 334111)
+8|SRID=3857;MULTIPOLYGON(((0 -6446276,8682920 -30240972,-8682920 -30240972,0 -6446276)),((11131949 0,0 -6446276,-11131949 0,11131949 0)))
+9|
+10|SRID=3857;POINT(-20037510 -30240972)
+COMMIT

lib/sql/test/CDB_UserTablesTest.sql

@@ -0,0 +1,32 @@
+SET client_min_messages TO ERROR;
+DO
+$do$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+$do$;
+SET client_min_messages TO NOTICE;
+
+CREATE TABLE pub(a int);
+CREATE TABLE prv(a int);
+GRANT SELECT ON TABLE pub TO publicuser;
+REVOKE SELECT ON TABLE prv FROM publicuser;
+SELECT CDB_UserTables() ORDER BY 1;
+SELECT 'all',CDB_UserTables('all') ORDER BY 2;
+SELECT 'public',CDB_UserTables('public') ORDER BY 2;
+SELECT 'private',CDB_UserTables('private') ORDER BY 2;
+SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2;
+-- now tests with public user
+\c contrib_regression publicuser
+SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2;
+SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2;
+SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2;
+\c contrib_regression postgres
+DROP TABLE pub;
+DROP TABLE prv;

lib/sql/test/CDB_UserTablesTest_expect

@@ -0,0 +1,19 @@
+SET
+DO
+SET
+CREATE TABLE
+CREATE TABLE
+GRANT
+REVOKE
+prv
+pub
+all|prv
+all|pub
+public|pub
+private|prv
+You are now connected to database "contrib_regression" as user "publicuser".
+all_publicuser|pub
+public_publicuser|pub
+You are now connected to database "contrib_regression" as user "postgres".
+DROP TABLE
+DROP TABLE

lib/sql/test/CDB_Username.sql

@@ -0,0 +1,23 @@
+SELECT session_user; -- postgres
+SELECT CDB_Username(); -- (NULL)
+
+-- Add the role fulano with api_key and connect with it
+\set QUIET on
+CREATE ROLE fulano LOGIN;
+GRANT USAGE ON SCHEMA cartodb TO fulano;
+GRANT EXECUTE ON FUNCTION CDB_Username() TO fulano;
+INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+SET SESSION AUTHORIZATION fulano;
+\set QUIET off
+
+SELECT session_user; -- fulano
+SELECT CDB_Username(); -- fulanito
+
+-- Remove fulano
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE USAGE ON SCHEMA cartodb FROM fulano;
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano;
+DROP ROLE fulano;
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano';
+\set QUIET off

lib/sql/test/CDB_Username_expect

@@ -0,0 +1,4 @@
+postgres
+
+fulano
+fulanito

lib/sql/test/CDB_XYZ_ExtentTest.sql

@@ -0,0 +1,9 @@
+
+WITH zoom AS ( select generate_series(0,2) as Z ),
+range AS ( select z, generate_series(0, pow(2,z)::int-1) as r FROM zoom),
+inp AS ( select z0.z, r1.r as x, r2.r as y FROM zoom z0, range r1, range r2 WHERE z0.z = r1.z and r1.z = r2.z ),
+ext AS ( select x,y,z,CDB_XYZ_Extent(x,y,z) as g from inp )
+select X::text || ',' || Y::text || ',' || Z::text as xyz,
+       round(st_xmin(g)), round(st_xmax(g)), round(st_ymin(g)), round(st_ymax(g))
+       from ext order by xyz;
+

lib/sql/test/CDB_XYZ_ExtentTest_expect

@@ -0,0 +1,21 @@
+0,0,0|-20037508|20037508|-20037508|20037508
+0,0,1|-20037508|0|0|20037508
+0,0,2|-20037508|-10018754|10018754|20037508
+0,1,1|-20037508|0|-20037508|0
+0,1,2|-20037508|-10018754|0|10018754
+0,2,2|-20037508|-10018754|-10018754|0
+0,3,2|-20037508|-10018754|-20037508|-10018754
+1,0,1|0|20037508|0|20037508
+1,0,2|-10018754|0|10018754|20037508
+1,1,1|0|20037508|-20037508|0
+1,1,2|-10018754|0|0|10018754
+1,2,2|-10018754|0|-10018754|0
+1,3,2|-10018754|0|-20037508|-10018754
+2,0,2|0|10018754|10018754|20037508
+2,1,2|0|10018754|0|10018754
+2,2,2|0|10018754|-10018754|0
+2,3,2|0|10018754|-20037508|-10018754
+3,0,2|10018754|20037508|10018754|20037508
+3,1,2|10018754|20037508|0|10018754
+3,2,2|10018754|20037508|-10018754|0
+3,3,2|10018754|20037508|-20037508|-10018754

lib/sql/test/Makefile

@@ -0,0 +1,5 @@
+all:
+	@echo "Try make check"
+
+check:
+	bundle exec rspec ../../../spec/lib/sql_test_spec.rb

lib/sql/test/README

@@ -0,0 +1,15 @@
+Adding tests consists in adding 2 files in this directory: one file
+containing the sql code and another containing the expected output.
+
+Example, to add a test for CDB_Something function, you'd add:
+
+ - CDB_SomethingTest.sql
+ - CDB_SomethingTest_expect
+
+To easy the generation of the expected file you can initially omit it,
+then run "make -C .. installcheck" from the top-level dir and copy
+../results/test/CDB_SomethingTest.out to CDB_SomethingTest_expect chopping
+off the first line:
+
+ make -C .. installcheck
+ tail -n +2 ../results/test/CDB_SomethingTest.out > CDB_SomethingTest_expect

lib/sql/test/extension/test.sh

@@ -0,0 +1,615 @@
+#!/bin/sh
+
+#
+# Tests for the extension since version 0.5.0. They don't replace SQL based ones, for now need to run both
+#
+
+# It is expected that you run this script as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_extension
+CMD='echo psql'
+CMD=psql
+SED=sed
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+
+function sql() {
+    local ROLE
+    local QUERY
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+    else
+      log_debug "Executing query '${QUERY}'"
+      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+    fi
+    CODERESULT=$?
+
+    echo ${RESULT}
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should-not" ]]
+    then
+        if [[ "${RESULT}" == "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' did not expect '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "GRANT USAGE ON SCHEMA cartodb TO ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}');"
+    sql "ALTER ROLE ${ROLE} SET search_path TO ${ROLE},cartodb,public;"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "REVOKE USAGE ON SCHEMA cartodb FROM ${ROLE};"
+    sql "DROP SCHEMA \"${ROLE}\" CASCADE;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+
+function create_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local RASTER_COL="the_raster_webmercator"
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${OVERVIEW_TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "SELECT AddOverviewConstraints('${ROLE}','${OVERVIEW_TABLENAME}','${RASTER_COL}','${ROLE}','${TABLENAME}','${RASTER_COL}',2);"
+}
+
+function drop_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "drop_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+
+    sql ${ROLE} "DROP TABLE ${ROLE}.${OVERVIEW_TABLENAME};"
+    sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
+}
+
+function setup_database() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    sql "CREATE EXTENSION postgis;"
+    sql postgres "DO
+\$\$
+BEGIN
+    IF substring(postgis_lib_version() FROM 1 FOR 1) = '3' THEN
+        CREATE EXTENSION postgis_raster;
+    END IF;
+END
+\$\$;"
+    sql "CREATE EXTENSION cartodb CASCADE;"
+    ${CMD} -c "ALTER DATABASE ${DATABASE} SET search_path = public, cartodb;"
+}
+
+function setup() {
+    setup_database
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+
+    create_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5), (6);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo;'
+
+    create_table cdb_testmember_2 bar
+    sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3);'
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+}
+
+
+function tear_down_database() {
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1 CASCADE;'
+    sql 'DROP SCHEMA cdb_testmember_2 CASCADE;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+
+    tear_down_database
+}
+
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        if [[ $# -eq 1 ]]
+        then
+            TESTS=`cat $0 | grep -o "$1[^\(]*"`
+        else
+            TESTS="$@"
+        fi
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+    setup
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+
+        clear_partial_result
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+    done
+    tear_down
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+#################################################### TESTS GO HERE ####################################################
+
+
+# Tests quota checking taking into account both geom and raster tables
+function test_quota_for_each_user() {
+    # Normal tables add 4096 bytes
+    # Raster tables no longer add anything so also count as 4096
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+
+    create_raster_table cdb_testmember_1 raster_1
+    create_raster_table cdb_testmember_2 raster_2
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 20480
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    create_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 36864
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    drop_raster_table cdb_testmember_1 raster_1
+    drop_raster_table cdb_testmember_2 raster_2
+    drop_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+}
+
+function test_cdb_tablemetadatatouch() {
+    sql postgres "CREATE TABLE touch_example (a int)"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should ''
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Another call doesn't fail
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT updated_at FROM CDB_TableMetadata WHERE tabname = 'touch_example'::regclass;" should-not ''
+
+    # Works with qualified tables
+    sql postgres "SELECT CDB_TableMetadataTouch('public.touch_example');"
+    sql postgres "SELECT CDB_TableMetadataTouch('public.\"touch_example\"');"
+    sql postgres "SELECT CDB_TableMetadataTouch('\"public\".touch_example');"
+    sql postgres "SELECT CDB_TableMetadataTouch('\"public\".\"touch_example\"');"
+
+    # Works with OID
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+    TABLE_OID=`${CMD} -U postgres ${DATABASE} -c "SELECT attrelid FROM pg_attribute WHERE attrelid = 'touch_example'::regclass limit 1;" -A -t`
+
+    # quoted OID works
+    sql postgres "SELECT CDB_TableMetadataTouch('${TABLE_OID}');"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    # non quoted OID works
+    sql postgres "SELECT CDB_TableMetadataTouch(${TABLE_OID});"
+    sql postgres "SELECT tabname from CDB_TableMetadata;" should 'touch_example'
+    sql postgres "SELECT count(*) from CDB_TableMetadata;" should 1
+
+    #### test tear down
+    sql postgres 'DROP TABLE touch_example;'
+}
+
+function test_cdb_tablemetadatatouch_fails_for_unexistent_table() {
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('unexistent_example');" fails
+}
+
+function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
+    sql postgres "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');" fails
+
+    sql postgres "GRANT ALL ON CDB_TableMetadata TO cdb_testmember_1;"
+    sql cdb_testmember_1 "SELECT CDB_TableMetadataTouch('touch_example');"
+
+    sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
+
+    #### test tear down
+    sql postgres 'DROP TABLE touch_example;'
+}
+
+function test_cdb_tablemetadatatouch_fully_qualifies_names() {
+    sql postgres "CREATE TABLE touch_invalidations (table_name text);"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+
+    #default schema
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "public.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+    sql postgres "DROP TABLE touch_example"
+
+    #setup different schema
+    sql postgres "CREATE SCHEMA test_schema;"
+    sql postgres "CREATE TABLE test_schema.touch_example (a int);"
+
+    #different schema outside search_path
+    sql postgres "SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #different schema in default search_path
+    sql postgres "SET search_path=test_schema,public,cartodb; SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #teardown different schema
+    sql postgres 'DROP TABLE test_schema.touch_example;'
+    sql postgres 'DROP SCHEMA test_schema;'
+
+
+
+    sql postgres 'DROP FUNCTION cartodb.cdb_invalidate_varnish(table_name text);'
+    sql postgres 'DROP TABLE touch_invalidations'
+}
+
+function test_cdb_tablemetadata_text() {
+
+    #create and touch tables
+    sql postgres "CREATE TABLE touch_ex_a (id int);"
+    sql postgres "CREATE TABLE touch_ex_b (id int);"
+    sql postgres "CREATE TABLE touch_ex_c (id int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_a');"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_b');"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_c');"
+
+    #ensure there is 1 record per table
+    QUERY="SELECT COUNT(1) FROM (SELECT 1 FROM cdb_tablemetadata_text "
+    QUERY+="GROUP BY tabname HAVING COUNT(1) > 1) s;"
+    sql postgres "$QUERY" should "0"
+
+    #ensure timestamps are distinct and properly ordered
+    QUERY="SELECT (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_a')"
+    QUERY+="    < (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_b');"
+    sql postgres "$QUERY" should "t"
+    QUERY="SELECT (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_b')"
+    QUERY+="    < (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_c');"
+    sql postgres "$QUERY" should "t"
+
+    #cleanup
+    sql postgres "DROP TABLE touch_ex_a;"
+    sql postgres "DROP TABLE touch_ex_b;"
+    sql postgres "DROP TABLE touch_ex_c;"
+
+}
+
+function test_cdb_column_names() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
+
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "carto"
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "db"
+
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames'::regclass) c) as s" should "carto"
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_2.table_cnames') c) as s" should "db"
+
+    # Using schema from owner
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" should "carto"
+
+    ## it's not possible to get column names from a table where you don't have permissions
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" fails
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_cnames'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_cnames'
+}
+
+function test_cdb_column_type() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_ctype(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_ctype(c text, a text, r text, t text, o text);'
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "integer"
+    sql cdb_testmember_2 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "text"
+
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_1.table_ctype', 'c')" should "integer"
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_2.table_ctype', 'c')" should "text"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_ctype'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_ctype'
+}
+
+function test_cdb_querytables_schema_and_table_names_with_dots() {
+    sql postgres 'CREATE SCHEMA "foo.bar";'
+    sql postgres 'CREATE TABLE "foo.bar"."c.a.r.t.o.d.b" (a int);'
+    sql postgres 'INSERT INTO "foo.bar"."c.a.r.t.o.d.b" values (1);'
+    sql postgres 'SELECT a FROM "foo.bar"."c.a.r.t.o.d.b";' should 1
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+
+    sql postgres 'DROP TABLE "foo.bar"."c.a.r.t.o.d.b";'
+    sql postgres 'DROP SCHEMA "foo.bar";'
+}
+
+function test_cdb_querytables_table_name_with_dots() {
+    sql postgres 'CREATE TABLE "w.a.d.u.s" (a int);';
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+
+    sql postgres 'DROP TABLE "w.a.d.u.s";';
+}
+
+function test_cdb_querytables_happy_cases() {
+    sql postgres 'CREATE TABLE wadus (a int);';
+    sql postgres 'CREATE TABLE "FOOBAR" (a int);';
+    sql postgres 'CREATE SCHEMA foo;'
+    sql postgres 'CREATE TABLE foo.wadus (a int);';
+
+    ## See how it does NOT quote anything here
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+
+    ## But it quotes when it's needed even if table name has no dots but was created with quotes
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "FOOBAR"$q$);' should '{"public.\"FOOBAR\""}'
+
+    sql postgres 'DROP TABLE wadus;'
+    sql postgres 'DROP TABLE "FOOBAR";'
+    sql postgres 'DROP TABLE foo.wadus;'
+    sql postgres 'DROP SCHEMA foo;'
+}
+
+function test_foreign_tables() {
+
+    DATABASE=fdw_target setup_database
+    DATABASE=fdw_target sql postgres "DO
+\$\$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+\$\$;"
+
+    DATABASE=fdw_target sql postgres 'CREATE SCHEMA test_fdw;'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo (a) values (42);'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo2 (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo2 (a) values (42);'
+    DATABASE=fdw_target sql postgres "CREATE USER fdw_user WITH PASSWORD 'foobarino';"
+    DATABASE=fdw_target sql postgres 'GRANT USAGE ON SCHEMA test_fdw TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo2 TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON cdb_tablemetadata_text TO fdw_user;'
+
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
+
+    # Add PGPORT to conf if it is set
+    PORT_SPEC=""
+    if [[ "$PGPORT" != "" ]] ; then
+        PORT_SPEC=", \"port\": \"$PGPORT\""
+    fi
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('fdws', '{\"test_fdw\": {\"server\": {\"host\": \"localhost\", \"dbname\": \"fdw_target\" $PORT_SPEC },
+                                           \"users\": {\"public\": {\"user\": \"fdw_user\", \"password\": \"foobarino\"}}}}')"
+
+    sql postgres "SELECT cartodb._CDB_Setup_FDW('test_fdw')"
+
+    sql postgres "SELECT cartodb.CDB_Add_Remote_Table('test_fdw', 'foo')"
+    sql postgres "SELECT * from test_fdw.foo;"
+
+
+    sql postgres "SELECT n.nspname,
+  c.relname,
+  s.srvname FROM pg_catalog.pg_foreign_table ft
+  INNER JOIN pg_catalog.pg_class c ON c.oid = ft.ftrelid
+  INNER JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+  INNER JOIN pg_catalog.pg_foreign_server s ON s.oid = ft.ftserver
+ORDER BY 1, 2" should "test_fdw|cdb_tablemetadata|test_fdw
+test_fdw|foo|test_fdw"
+
+    sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo'::regclass) < NOW()" should 't'
+
+    sql postgres "SELECT a from test_fdw.foo LIMIT 1;" should 42
+
+    # Check function CDB_QueryTables_Updated_At
+    sql postgres 'CREATE TABLE local (b int);'
+    sql postgres 'INSERT INTO local (b) VALUES (43);'
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local'::regclass);"
+    local query='$query$ SELECT * FROM test_fdw.foo, local $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname;" should 'fdw_target|test_fdw|foo
+test_extension|public|local'
+    sql postgres "SELECT table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) order by updated_at;" should 'foo
+local'
+
+    # Check function CDB_Last_Updated_Time
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) > (now() - interval '1 minute')" should 't'
+
+    # Check we quote names on output as needed (as CDB_QueryTablesText does)
+    sql postgres 'CREATE TABLE "local-table-with-dashes" (c int)';
+    sql postgres 'INSERT INTO "local-table-with-dashes" (c) VALUES (44)';
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local-table-with-dashes'::regclass);"
+    query='$query$ SELECT * FROM test_fdw.foo, local, public."local-table-with-dashes" $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname, schema_name, table_name;" should 'fdw_target|test_fdw|foo
+test_extension|public|local
+test_extension|public|"local-table-with-dashes"'
+
+    # Check CDB_Last_Updated_Time supports quoted identifiers
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) > (now() - interval '1 minute')" should 't'
+
+    # Check CDB_Get_Foreign_Updated_At is robust to unimported CDB_TableMetadata
+    sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;"
+    sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't'
+
+    # Teardown
+    DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo2 FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
+
+    sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
+    DATABASE=fdw_target tear_down_database
+}
+
+function test_cdb_catalog_basic_node() {
+    DEF="'{\"type\":\"buffer\",\"source\":\"b2db66bc7ac02e135fd20bbfef0fdd81b2d15fad\",\"radio\":10000}'"
+    sql postgres "INSERT INTO cartodb.cdb_analysis_catalog (node_id, analysis_def) VALUES ('1bbc4c41ea7c9d3a7dc1509727f698b7', ${DEF}::json)"
+    sql postgres "SELECT status from cartodb.cdb_analysis_catalog where node_id = '1bbc4c41ea7c9d3a7dc1509727f698b7'" should 'pending'
+    sql postgres "DELETE FROM cartodb.cdb_analysis_catalog"
+}
+
+#################################################### TESTS END HERE ####################################################
+
+run_tests $@
+
+exit ${OK}

lib/sql/test/organization/test.sh

@@ -0,0 +1,631 @@
+#!/bin/sh
+
+#
+# It is expected that you run this script
+# as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_organizations
+CMD=psql
+SED=sed
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+function sql() {
+    local ROLE
+    local QUERY
+    ERROR_OUTPUT_FILE='/tmp/test_error.log'
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+        log_debug "Executing query '${QUERY}' as ${ROLE}"
+        RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
+    else
+        log_debug "Executing query '${QUERY}'"
+        RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
+    fi
+    CODERESULT=$?
+    ERROR_OUTPUT=`cat "${ERROR_OUTPUT_FILE}"`
+    rm ${ERROR_OUTPUT_FILE}
+
+    echo -n "> Code Result: "
+    echo -n ${CODERESULT}
+    echo -n "; Result: "
+    echo -n ${RESULT}
+    echo -n "; Error output: "
+    echo -n ${ERROR_OUTPUT}
+
+    # Some warnings should actually be failures
+    if [[ ${CODERESULT} == "0" ]]
+    then
+        case "${ERROR_OUTPUT}" in
+            WARNING:*no*privileges*were*granted*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES GRANTING WARNING"
+                CODERESULT=1
+            ;;
+            WARNING:*no*privileges*could*be*revoked*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES REVOKING WARNING"
+                CODERESULT=1
+            ;;
+            *) ;;
+        esac
+        echo -n "; Code result after warnings: "
+        echo -n ${CODERESULT}
+    fi
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}')"
+    sql "ALTER ROLE ${ROLE} SET search_path TO ${ROLE},cartodb,public;"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+function truncate_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "truncate_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "TRUNCATE TABLE ${ROLE}.${TABLENAME};"
+}
+
+
+function setup() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    ${CMD} -c "ALTER DATABASE ${DATABASE} SET search_path = public, cartodb;"
+    sql "CREATE EXTENSION cartodb CASCADE;"
+    ${CMD} -c "ALTER DATABASE ${DATABASE} SET search_path = public, cartodb;"
+
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_org_admin
+    sql "SELECT cartodb.CDB_Organization_AddAdmin('cdb_org_admin');"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+    sql postgres "DO
+\$\$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+\$\$;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO publicuser;"
+
+    create_table cdb_testmember_1 foo
+    create_table cdb_testmember_2 bar
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_a_tmp')"
+    sql "SELECT cartodb.CDB_Group_RenameGroup('group_a_tmp', 'group_a')"
+
+    sql "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_b')"
+}
+
+
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "select cartodb.CDB_Group_DropGroup('group_b')"
+
+    sql "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "select cartodb.CDB_Group_DropGroup('group_a')"
+    sql "SELECT cartodb.CDB_Organization_RemoveAdmin('cdb_org_admin');"
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1;'
+    sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_org_admin;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_org_admin;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+    sql 'DROP ROLE cdb_org_admin;'
+
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        TESTS="$@"
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+
+    setup
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+        clear_partial_result
+        log_info "############################# TESTS #############################"
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+    done
+    tear_down
+
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+
+#################################################### TESTS GO HERE ####################################################
+
+function test_member_2_cannot_read_without_permission() {
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+}
+
+function test_member_1_cannot_grant_read_permission_to_other_schema_than_its_one() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_2', 'foo', 'cdb_testmember_2')" fails
+}
+
+function test_member_1_grants_read_permission_and_member_2_can_read() {
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.bar;' fails
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+}
+
+function test_member_2_cannot_add_table_to_member_1_schema_after_table_permission_added() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 "CREATE TABLE cdb_testmember_1.bar ( a int );" fails
+}
+
+function test_grant_read_permission_between_two_members() {
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.bar VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.bar;' should 5
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+    truncate_table cdb_testmember_2 bar
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1')"
+}
+
+function test_member_2_cannot_write_to_member_1_table() {
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails
+}
+
+function test_member_1_cannot_grant_read_write_permission_to_other_schema_than_its_one() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_2', 'foo', 'cdb_testmember_2')" fails
+}
+
+function test_member_2_can_write_to_member_1_table_after_write_permission_is_added() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+}
+
+function test_member_2_can_write_to_member_1_table_and_sequence_after_write_permission_is_added() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;"
+
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+}
+
+function test_member_2_can_write_to_member_1_table_with_non_sequence_cartodb_id_after_write_permission_is_added() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id INTEGER;"
+
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+}
+
+function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table() {
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9), (10);'
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 6
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_1.foo;' fails
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+}
+
+function test_member_1_removes_access_and_member_2_can_no_longer_write_to_the_table() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails
+
+    # Cleanup
+    truncate_table cdb_testmember_1 foo
+}
+
+function test_giving_permissions_to_two_tables_and_removing_from_first_table_should_not_remove_from_second() {
+    #### test setup
+    # create an extra table for cdb_testmember_1
+    create_table cdb_testmember_1 foo_2
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4);'
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo_2 VALUES (1), (2), (3), (4), (5);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo_2;'
+
+    # gives read permission to both tables
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo_2', 'cdb_testmember_2')"
+
+    # cdb_testmember_2 has access to both tables
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo_2;' should 5
+
+    # cdb_testmember_1 removes access to foo table
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+
+    # cdb_testmember_2 should have access to foo_2 table but not to table foo
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo_2;' should 5
+
+
+    #### test tear down
+    truncate_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo_2;'
+}
+
+function test_cdb_org_member_role_allows_reading_to_all_users_without_explicit_permission() {
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4);'
+
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Add_Table_Organization_Read_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+
+    # Cleanup
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Remove_Organization_Access_Permission('cdb_testmember_1', 'foo');"
+    truncate_table cdb_testmember_1 foo
+}
+
+function test_user_can_read_when_it_has_permission_after_organization_permission_is_removed() {
+    create_role_and_schema cdb_testmember_3
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4);'
+
+    # shares with cdb_testmember_2 and can read but cdb_testmember_3 cannot
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+
+    # granting to organization allows to read to both: cdb_testmember_2 and cdb_testmember_3
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Add_Table_Organization_Read_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+
+    # removing access from organization should keep permission on cdb_testmember_2 but drop it to cdb_testmember_3
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Remove_Organization_Access_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 4
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+
+    # Cleanup
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    truncate_table cdb_testmember_1 foo
+    drop_role_and_schema cdb_testmember_3
+}
+
+function test_cdb_querytables_returns_schema_and_table_name() {
+    sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
+}
+
+function test_cdb_querytables_works_with_parentheses() {
+    sql cdb_testmember_1 "select * from CDB_QueryTables('(select * from foo)');" should "{cdb_testmember_1.foo}"
+}
+
+function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
+function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
+function test_cdb_usertables_should_work_with_orgusers() {
+
+    # This test validates the changes proposed in https://github.com/CartoDB/cartodb/pull/5021
+
+    # create tables
+    sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
+    sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
+    sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
+
+    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+
+
+    # this is what we need to make public tables available in CDB_UserTables
+    sql postgres "grant publicuser to cdb_testmember_1;"
+    sql postgres "grant publicuser to cdb_testmember_2;"
+
+
+    # this is required to enable select from other schema
+    sql postgres "GRANT USAGE ON SCHEMA cdb_testmember_1 TO publicuser";
+
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('all')" should 1
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('public')" should 1
+    sql publicuser "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    sql publicuser "SELECT * FROM CDB_UserTables('all')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('private')" should ""
+    # the following tests are for https://github.com/CartoDB/cartodb-postgresql/issues/98
+    # cdb_testmember_2 is already owner of `bar` table
+    sql cdb_testmember_2 "select string_agg(t,',') from (select cdb_usertables('all') t order by t) as s" should "bar,test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('private')" should "bar"
+
+    # test cdb_testmember_2 can select from cdb_testmember_1's public table
+    sql cdb_testmember_2 "SELECT * FROM cdb_testmember_1.test_perms_pub" should 1
+
+    sql postgres 'REVOKE USAGE ON SCHEMA cdb_testmember_1 FROM publicuser;'
+    sql cdb_testmember_1 "DROP TABLE test_perms_pub"
+    sql cdb_testmember_1 "DROP TABLE test_perms_priv"
+}
+
+function test_CDB_Group_Table_GrantRead_should_grant_select_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_CDB_Group_Table_GrantReadWrite_should_grant_insert_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_cant_be_used_by_normal_members() {
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_CreateGroup('group_x_1');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RenameGroup('group_a', 'group_x_2');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_DropGroup('group_a');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_2']);" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1']);" fails
+}
+
+function test_group_permission_functions_cant_be_used_by_normal_members() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_can_be_used_by_org_admin() {
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_CreateGroup('group_x_tmp');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RenameGroup('group_x_tmp', 'group_x');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_AddUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RemoveUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    # TODO: workaround superadmin limitation
+    sql "SELECT cartodb.CDB_Group_DropGroup('group_x');"
+}
+
+function test_org_admin_cant_grant_permissions_on_tables_he_does_not_own() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_valid_group_names() {
+    sql postgres "select cartodb._CDB_Group_GroupRole('group_1$_a');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('GROUP_1$_A');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('_group_1$_a');"
+}
+
+function test_administrator_name_generation() {
+    sql postgres "select cartodb._CDB_Organization_Admin_Role_Name();"
+}
+
+function test_conf() {
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('test_conf', '{ \"a_key\": \"test_val\" }')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should '{ "a_key": "test_val" }'
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_RemoveConf('test_conf')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+}
+
+#################################################### TESTS END HERE ####################################################
+
+
+
+run_tests $@
+
+exit ${OK}

lib/sql/test/overviews/gen_points.rb

@@ -0,0 +1,43 @@
+# Ruby script to generate test/overviews/fixtures.sql for testing overviews
+# Generated tables:
+# * base_bare_t -- points without attributes (only PK, geometries)
+
+NUM_CLUSTERS = 128
+MAX_PER_CLUSTER = 16
+CLUSTER_RADIUS = 1E-3
+MIN_X = -10.0
+MAX_X =  10.0
+MIN_Y =  30.0
+MAX_Y = 40.0
+ATTRIBUTES = "number double precision, int_number integer, name text, start date"
+
+id = 0
+POINTS = (0...NUM_CLUSTERS).map{
+  x = MIN_X + rand()*(MAX_X - MIN_X)
+  y = MIN_Y + rand()*(MAX_Y - MIN_Y)
+  (0..rand(MAX_PER_CLUSTER)).map{
+    id += 1
+    {
+      id: id,
+      x: (x + rand()*CLUSTER_RADIUS).round(6),
+      y: (y + rand()*CLUSTER_RADIUS).round(6)
+    }
+  }
+}.flatten
+
+values = POINTS.map{ |point|
+  "#{point[:id]}, 'SRID=4326;POINT(#{point[:x]} #{point[:y]})'::geometry, ST_Transform('SRID=4326;POINT(#{point[:x]} #{point[:y]})'::geometry, 3857)"
+}
+
+File.open('fixtures.sql', 'w') do |sql|
+
+  sql.puts "-- bare table with no attribute columns"
+  sql.puts "CREATE TABLE base_bare_t (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry);"
+  sql.puts "INSERT INTO base_bare_t VALUES"
+  sql.puts values.map{|v| "(#{v})"}.join(",\n") + ";"
+
+  sql.puts "-- table with attributes"
+  sql.puts "CREATE TABLE base_t (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry, #{ATTRIBUTES});"
+  sql.puts "INSERT INTO base_t VALUES"
+  sql.puts values.map{|v| "(#{v})"}.join(",\n") + ";"
+end

lib/sql/test/perf/CDB_HexagonGridPerf.sql

@@ -0,0 +1,6 @@
+-- 25690 cells covering full webmercator extent
+-- real    0m9.448s -- as of e0e76843f785a420c277a6fb2f762601570ffb98
+-- real    0m0.243s -- as of 50c487b83837e4d5216fcc19d637dd8db6baa44a
+
+SELECT count(*) FROM (
+ SELECT CDB_HexagonGrid(ST_MakeEnvelope(-20194051, -20194051, 20194051, 20194051), 156543) ) f;

lib/sql/test/perf/CDB_TransformToWebmercatorPerf.sql

@@ -0,0 +1,16 @@
+-- Run psql with -tA switches and expect
+-- CDB_TransformToWebmercatorTest_expect
+select '1', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;LINESTRING(90 90, 0 80)'), 1));
+select '2', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;LINESTRING(90 90, 0 90)'), 1));
+select '3', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;POINT(0 90)'), 1));
+select '4', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTIPOINT(10 3, 0 90, 0 4)'), 1));
+select '5', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTIPOINT(10 3)'), 1));
+select '6', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;MULTILINESTRING((90 90, 0 90),(0 4, -4 5))'), 1));
+select '7', ST_AsEWKT(ST_SnapToGrid(CDB_TransformToWebmercator(
+ 'SRID=4326;POINT(5 3)'), 1));