Release 0.10.0

Quote schema and table names returned by CDB_QueryTables

CONTRIBUTING.md

@@ -49,12 +49,16 @@ also installs migration scripts to go from "V" to "V"next and from "V"next
 to "V". Example to upgrade a 0.2.0dev version:
 
 ```sql
-ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
+ALTER EXTENSION cartodb UPDATE TO '0.2.0next';
 ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
 ```
-
 Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
 
 ```sql
 SELECT cartodb.cdb_extension_reload();
 ```
+
+A useful query:
+```sql
+SELECT * FROM pg_extension_update_paths('cartodb') WHERE path IS NOT NULL AND source = cdb_version();
+```

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.8.2
+EXTVERSION = 0.10.0
 
 SED = sed
 
@@ -41,6 +41,13 @@ UPGRADABLE = \
   0.7.4 \
   0.8.0 \
   0.8.1 \
+  0.8.2 \
+  0.9.0 \
+  0.9.1 \
+  0.9.2 \
+  0.9.3 \
+  0.9.4 \
+  0.10.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,45 @@
+0.10.0 (2015-09-07)
+-----------------
+* Quote schema and table names returned by CDB_QueryTables [#134](https://github.com/CartoDB/cartodb-postgresql/pull/134). Use quote_ident to quote schema and table names when necessary.
+* Fixed CDB_ColumnNames [#122](https://github.com/CartoDB/cartodb-postgresql/issues/122) and CDB_ColumnType [#130](https://github.com/CartoDB/cartodb-postgresql/issues/130) should honor regclass, returning columns for just the table in the schema and not in any other one [#131](https://github.com/CartoDB/cartodb-postgresql/pull/131).
+* Add kurtosis and skewness [#124](https://github.com/CartoDB/cartodb-postgresql/pull/124).
+* Removed `DROP FUNCTION IF EXISTS cdb_usertables(text);` [#129](https://github.com/CartoDB/cartodb-postgresql/pull/129). This was needed for upgrading between 0.7.4 to 0.8.0 but is no longer needed.
+
+0.9.4 (2015-08-28)
+------------------
+* Fixed issue with indices when renaming tables [#123](https://github.com/CartoDB/cartodb-postgresql/issues/123)
+
+0.9.3 (2015-08-27)
+------------------
+* Modify sampling of quota trigger [#126](https://github.com/CartoDB/cartodb-postgresql/issues/126)
+
+0.9.2 (2015-08-24)
+------------------
+* Fix for `the_geom` column present but not SRID (EWKT) and other corner cases [#121](https://github.com/CartoDB/cartodb-postgresql/pull/121)
+
+0.9.1 (2015-08-19)
+------------------
+* Fix for transformation to webmercator in corner cases [#116](https://github.com/CartoDB/cartodb-postgresql/issues/116)
+
+0.9.0 (2015-08-19)
+------------------
+* Re-implementation of `CDB_CartodbfyTable` functions
+  - The signature of the main function changes to
+    ```
+    FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
+    RETURNS REGCLASS
+    ```
+    - The `destschema` does not need to match the origin schema of `reloid`
+	- It returns the `regclass` of the cartodbfy'ed table, if it needs to be rewritten.
+  - There are many optimizations
+  - The columns `created_at` and `updated_at` will no longer be added
+* Fix for CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` #110
+* Review test to validate permissions in public tables [#112](https://github.com/CartoDB/cartodb-postgresql/pull/112)
+
+0.8.3 (2015-08-14)
+------------------
+* Fixes CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` [#108](https://github.com/CartoDB/cartodb-postgresql/issues/108)
+
 0.8.2 (2015-07-27)
 ------------------
 * Fix for CDB_UserTables returning wrong listings when publicuser is used

README.md

@@ -37,6 +37,8 @@ NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
 
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
+      
+NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
 ---------------

expected/test_ddl_triggers.out

@@ -25,6 +25,8 @@ SELECT CDB_SetUserQuotaInBytes('c', 0);
                        0
 (1 row)
 
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
+NOTICE:  role "cartodb_postgresql_unpriv_user" does not exist, skipping
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -36,7 +38,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
@@ -45,17 +46,6 @@ NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i 
-------------+-----+------+----------+----------------------+---
-          1 | t   | t    |          |                      | 1
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -72,28 +62,14 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as cartodb_id INTO c.t4;
 NOTICE:  trigger "track_updates" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t4" does not exist, skipping
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
-NOTICE:  Trying to recover data from _cartodb_id0 column
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t4) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator 
-------------+-----+------+----------+----------------------
-          1 | t   | t    |          | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -115,27 +91,12 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc 
-------------+-----+------+----------+----------------------+---+---------
-          1 | t   | t    |          |                      | 1 | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -154,27 +115,12 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -196,27 +142,12 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 drop column the_geom_webmercator;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -239,17 +170,6 @@ select pg_sleep(.1);
 
 alter table c.t3 add column id2 int;
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 | id2 
-------------+-----+------+----------+----------------------+---+---------+----------+-----
-          1 | t   | t    |          |                      | 1 |         |          |    
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -272,5 +192,6 @@ select count(*) from CDB_TableMetadata;
      0
 (1 row)
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

scripts-available/CDB_ColumnNames.sql

@@ -3,11 +3,12 @@ CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
 
-    SELECT column_name 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || '';
+    SELECT c.column_name
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_ColumnType.sql

@@ -3,12 +3,13 @@ CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
 
-    SELECT data_type 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || ''
-        AND column_name = '' || quote_ident($2) || '';
+    SELECT c.data_type
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND column_name = $2
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_DDLTriggers.sql

@@ -4,17 +4,23 @@ CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
 RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
-  rel_namespace TEXT;
+  rel RECORD;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_relation_create_eventinfo();
 
   -- We're only interested in real relations
   IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid=(event_info.new).relnamespace INTO rel_namespace;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relname::TEXT, rel_namespace, (event_info.new).relnamespace;
+	 event_info.relation, rel.relkind, rel.relname, rel.nspname, rel.relnamespace;
 
   -- We don't want to react to alters triggered by superuser,
   IF current_setting('is_superuser') = 'on' THEN
@@ -25,15 +31,15 @@ BEGIN
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
   -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   RAISE DEBUG 'Inserting into cartodb.CDB_TableMetadata';
 
   -- Add entry to CDB_TableMetadata (should CartodbfyTable do this?)
-  INSERT INTO cartodb.CDB_TableMetadata(tabname,updated_at) 
-  VALUES (event_info.relation, now());
+  INSERT INTO cartodb.CDB_TableMetadata(tabname, updated_at) 
+  VALUES (newtable, now());
 
 END; $$;
 -- }
@@ -66,14 +72,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
-  rel_namespace TEXT;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_alter_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % altered by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -84,16 +95,14 @@ BEGIN
     RETURN;
   END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
-
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
   WHERE tabname = event_info.relation; 
 
 END; $$;
@@ -106,14 +115,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
-  rel_namespace TEXT;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_drop_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % drop by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -124,17 +138,16 @@ BEGIN
     RETURN;
   END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
 
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
+  WHERE tabname = event_info.relation;
 
 END; $$;
 -- }
@@ -149,10 +162,15 @@ DECLARE
 BEGIN
   event_info := schema_triggers.get_column_add_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % added by % (superuser? %) in relation % of kind %',
-	 (event_info.new).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.new).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;

scripts-available/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;

scripts-available/CDB_Quota.sql

@@ -1,3 +1,19 @@
+CREATE OR REPLACE FUNCTION cartodb._CDB_total_relation_size(_schema_name TEXT, _table_name TEXT)
+RETURNS bigint AS
+$$
+DECLARE relation_size bigint := 0;
+BEGIN
+  BEGIN
+    SELECT pg_total_relation_size(format('"%s"."%s"', _schema_name, _table_name)) INTO relation_size;
+  EXCEPTION
+    WHEN undefined_table OR OTHERS THEN
+      RAISE NOTICE 'cartodb._CDB_total_relation_size(''%'', ''%'') caught error: % (%)', _schema_name, _table_name, SQLERRM, SQLSTATE;
+  END;
+  RETURN relation_size;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;
+
 -- Return the estimated size of user data. Used for quota checking.
 CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
 RETURNS bigint AS
@@ -24,7 +40,7 @@ BEGIN
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(pg_total_relation_size('"' || schema_name || '"."' || table_name || '"')))) table_size,
+    SELECT COALESCE(INT8(SUM(cartodb._CDB_total_relation_size(schema_name, table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1

scripts-available/CDB_Stats.sql

@@ -0,0 +1,47 @@
+--
+-- Calculate basic statistics of a given dataset
+--
+-- @param in_array A numeric array of numbers
+--
+-- Returns: statistical quantity chosen
+-- 
+-- References: http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm
+--
+
+-- Calculate kurtosis
+CREATE OR REPLACE FUNCTION CDB_Kurtosis ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    k numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 4)) / ( $2 * power($3, 4)) - 3
+             FROM (SELECT unnest($4) e ) x'
+    INTO k
+    USING a, c, s, in_array;
+
+    RETURN k;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Calculate skewness
+CREATE OR REPLACE FUNCTION CDB_Skewness ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    sk numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 * power($3, 3))
+             FROM (SELECT unnest($4) e ) x'
+    INTO sk
+    USING a, c, s, in_array;
+
+    RETURN sk;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_UserTables.sql

@@ -5,7 +5,6 @@
 --
 -- Currently accepted permissions are: 'public', 'private' or 'all'
 --
-DROP FUNCTION IF EXISTS cdb_usertables(text);
 CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
 RETURNS SETOF name
 AS $$

scripts-enabled/CDB_Stats.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Stats.sql

sql/test_ddl_triggers.sql

@@ -10,6 +10,7 @@ create schema c;
 
 SELECT CDB_SetUserQuotaInBytes('c', 0);
 
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -21,13 +22,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -38,11 +32,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
 
 RESET SESSION AUTHORIZATION;
 select
@@ -58,13 +47,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -75,13 +57,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -95,13 +70,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 drop column the_geom_webmercator;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -115,13 +83,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 add column id2 int;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -136,5 +97,6 @@ RESET SESSION AUTHORIZATION;
 drop schema c cascade;
 select count(*) from CDB_TableMetadata;
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

sql/test_setup.sql

@@ -7,4 +7,4 @@ RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
-$$ LANGUAGE 'plpgsql';
+$$ LANGUAGE 'plpgsql';

test/CDB_CartodbfyTableTest.sql

@@ -12,6 +12,7 @@ DECLARE
   tmp INTEGER;
   ogc_geom geometry_columns; -- old the_geom record in geometry_columns
   ogc_merc geometry_columns; -- old the_geom_webmercator record in geometry_columns
+  tabtext TEXT;
 BEGIN
 
   -- Save current constraints on geometry columns, if any
@@ -30,24 +31,17 @@ BEGIN
     END IF;
   END LOOP;
 
+  tabtext := Format('%s.%s','public',tabname);
+  RAISE NOTICE 'CARTODBFYING % !!!!', tabtext;
   PERFORM CDB_CartodbfyTable('public', tabname);
+  tabname := tabtext::regclass;
 
   sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
   EXECUTE sql INTO STRICT id;
-  sql := 'SELECT created_at,updated_at,the_geom_webmercator FROM '
+  sql := 'SELECT the_geom_webmercator FROM '
     || tabname::text || ' WHERE cartodb_id = ' || id;
   EXECUTE sql INTO STRICT rec;
 
-  -- Check created_at and updated_at at creation time
-  lag = rec.created_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'created_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-  lag = rec.updated_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'updated_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-
   -- Check the_geom_webmercator trigger
   IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
     RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
@@ -84,7 +78,7 @@ BEGIN
         rec.f_geometry_column, rec.srid, rec.expsrid;
     END IF;
     -- Check TYPE constraint didn't change
-    IF rec.type != rec.exptype THEN
+    IF (rec.type != 'GEOMETRY') AND (rec.type != 'POINT') THEN
       RAISE EXCEPTION 'type of % in geometry_columns is %, expected %',
         rec.f_geometry_column, rec.type, rec.exptype;
     END IF;
@@ -115,10 +109,10 @@ BEGIN
   -- Check null constraint on cartodb_id, created_at, updated_at
   SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
     AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
-      ( 'cartodb_id', 'created_at', 'updated_at' )
+      ( 'cartodb_id' )
     AND NOT a.attnotnull INTO strict tmp;
   IF tmp > 0 THEN
-      RAISE EXCEPTION 'cartodb_id or created_at or updated_at are missing not-null constraint';
+      RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
   END IF;
 
   -- Cleanup
@@ -170,34 +164,48 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- table with existing updated_at and created_at fields ot type text
-CREATE TABLE t AS SELECT NOW()::text as created_at,
-                         NOW()::text as updated_at,
-                         NOW() as reftime;
-SELECT CDB_CartodbfyTableCheck('t', 'text timestamps');
-SELECT extract(secs from reftime-created_at),
-       extract(secs from reftime-updated_at) FROM t;
-CREATE VIEW v AS SELECT * FROM t;
-SELECT CDB_CartodbfyTableCheck('t', 'cartodbfied with view');
-DROP VIEW v;
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text
+-- CREATE TABLE t AS SELECT 10::text as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+-- select cartodb_id/2 FROM t;
+-- DROP TABLE t;
 
--- table with existing cartodb_id field of type text
-CREATE TABLE t AS SELECT 10::text as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
-select cartodb_id/2 FROM t; 
-DROP TABLE t;
-
--- table with existing cartodb_id field of type text not casting
-CREATE TABLE t AS SELECT 'nan' as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
-select cartodb_id,_cartodb_id0 FROM t; 
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text not casting
+-- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+-- select cartodb_id,_cartodb_id0 FROM t;
+-- DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
 SELECT CDB_CartodbfyTableCheck('t', 'unsequenced cartodb_id');
-select cartodb_id FROM t; 
+SELECT cartodb_id FROM t; 
+DROP TABLE t;
+
+-- table with text geometry column
+CREATE TABLE t AS SELECT 'SRID=4326;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column');
+SELECT cartodb_id FROM t; 
+DROP TABLE t;
+
+-- table with text geometry column, no SRS
+CREATE TABLE t AS SELECT 'POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, no srs');
+SELECT cartodb_id FROM t; 
+DROP TABLE t;
+
+-- table with text geometry column, unusual SRS
+CREATE TABLE t AS SELECT 'SRID=26910;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, srs = 26819');
+SELECT cartodb_id FROM t; 
+DROP TABLE t;
+
+-- table with text unparseable geometry column
+CREATE TABLE t AS SELECT 'SRID=26910;PONT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, unparseable content');
+SELECT cartodb_id FROM t; 
 DROP TABLE t;
 
 -- table with existing cartodb_id serial primary key
@@ -208,21 +216,14 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
 AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
 DROP TABLE t;
 
--- table with existing the_geom and created_at and containing null values
--- Really, a test for surviving an longstanding PostgreSQL bug:
--- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
-CREATE TABLE t (
-    the_geom geometry(Geometry,4326),
-    created_at timestamptz,
-    updated_at timestamptz
-);
-COPY t (the_geom, created_at, updated_at) FROM stdin;
-0106000020E610000001000000010300000001000000050000009EB8244146435BC017B65E062AD343409EB8244146435BC0F51AF6E2708044400B99891683765AC0F51AF6E2708044400B99891683765AC017B65E062AD343409EB8244146435BC017B65E062AD34340	2012-06-06 21:59:08	2013-06-10 20:17:20
-0106000020E61000000100000001030000000100000005000000DA7763431A1A5CC0FBCEE869313C3A40DA7763431A1A5CC09C1B8F55BC494440F9F4A9C7993356C09C1B8F55BC494440F9F4A9C7993356C0FBCEE869313C3A40DA7763431A1A5CC0FBCEE869313C3A40	2012-06-06 21:59:08	2013-06-10 20:17:20
-\N	\N	\N
-\.
-SELECT CDB_CartodbfyTableCheck('t', 'null geom and timestamp values');
-DROP TABLE t;
+-- tables can be renamed and there's no index name clashing #123
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+ALTER TABLE original RENAME TO original_renamed;
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+DROP TABLE original_renamed;
+DROP TABLE original;
 
 -- TODO: table with existing custom-triggered the_geom
 

test/CDB_CartodbfyTableTest_expect

@@ -2,8 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 7 at PERFORM
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
@@ -28,30 +28,35 @@ trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
 DROP TABLE
 SELECT 1
-text timestamps cartodbfied fine
-0|0
-CREATE VIEW
-cartodbfied with view cartodbfied fine
-DROP VIEW
-DROP TABLE
-SELECT 1
-text cartodb_id cartodbfied fine
-5
-DROP TABLE
-SELECT 1
-uncasting text cartodb_id cartodbfied fine
-1|nan
-DROP TABLE
-SELECT 1
 unsequenced cartodb_id cartodbfied fine
 1
 DROP TABLE
+SELECT 1
+text the_geom column cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, no srs cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, srs = 26819 cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, unparseable content cartodbfied fine
+1
+DROP TABLE
 CREATE TABLE
 cartodb_id serial primary key cartodbfied fine
 t_pkey|cartodb_id
 DROP TABLE
 CREATE TABLE
-null geom and timestamp values cartodbfied fine
+original
+ALTER TABLE
+CREATE TABLE
+original
+DROP TABLE
 DROP TABLE
 DROP FUNCTION
 DROP FUNCTION

test/CDB_QuotaTest.sql

@@ -1,22 +1,31 @@
 set client_min_messages to error;
 \set VERBOSITY default
 
+-- See the dice
+SELECT setseed(0.5);
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
       EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(2049,4096);
+INSERT INTO big SELECT generate_series(4097,6144);
+INSERT INTO big SELECT generate_series(6145,8192);
+-- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
+-- END Test for #108
+SELECT setseed(0.9);
 SELECT CDB_SetUserQuotaInBytes(2);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8194);
 DROP TABLE big;
 set client_min_messages to NOTICE;
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,15 +1,20 @@
 SET
+
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
+big
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+454656
+909312
+0
 
-INSERT 0 2048
-INSERT 0 2048
-INSERT 0 2048
 2
-ERROR:  Quota exceeded by 567.998046875KB
+ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/CDB_StatsTest.sql

@@ -0,0 +1,16 @@
+-- continuous uniform distribution has kurtosis = -6/5, skewness = 0.0
+-- http://mathworld.wolfram.com/UniformDistribution.html
+set client_min_messages to ERROR;
+
+With dist As (
+  SELECT random()::numeric As val
+  FROM generate_series(1,50000) t
+)
+
+SELECT 
+  -- does random dist values match within 1% of known values
+  abs(CDB_Kurtosis(array_agg(val)) + 1.20) < 1e-2 As kurtosis,
+  abs(CDB_Skewness(array_agg(val)) - 0) < 1e-2 As skewness
+FROM dist;
+
+set client_min_messages to NOTICE;

test/CDB_StatsTest_expect

@@ -0,0 +1,3 @@
+SET
+t|t
+SET

test/extension/run_at_cartodb_schema.sql

@@ -1,4 +1,6 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
 \i scripts-available/CDB_TableMetadata.sql
+\i scripts-available/CDB_ColumnNames.sql
+\i scripts-available/CDB_ColumnType.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -178,6 +178,7 @@ function setup() {
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
     sql "CREATE EXTENSION postgis;"
+    sql "CREATE EXTENSION plpythonu;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
@@ -227,7 +228,12 @@ function run_tests() {
     local TESTS
     if [[ $# -ge 1 ]]
     then
-        TESTS="$@"
+        if [[ $# -eq 1 ]]
+        then
+            TESTS=`cat $0 | grep -o "$1[^\(]*"`
+        else
+            TESTS="$@"
+        fi
     else
         TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
     fi
@@ -337,6 +343,92 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_column_names() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
+
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "carto"
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "db"
+
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames'::regclass) c) as s" should "carto"
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_2.table_cnames') c) as s" should "db"
+
+    # Using schema from owner
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" should "carto"
+
+    ## it's not possible to get column names from a table where you don't have permissions
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" fails
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_cnames'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_cnames'
+}
+
+function test_cdb_column_type() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_ctype(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_ctype(c text, a text, r text, t text, o text);'
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "integer"
+    sql cdb_testmember_2 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "text"
+
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_1.table_ctype', 'c')" should "integer"
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_2.table_ctype', 'c')" should "text"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_ctype'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_ctype'
+}
+
+function test_cdb_querytables_schema_and_table_names_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE SCHEMA "foo.bar";'
+    sql postgres 'CREATE TABLE "foo.bar"."c.a.r.t.o.d.b" (a int);'
+    sql postgres 'INSERT INTO "foo.bar"."c.a.r.t.o.d.b" values (1);'
+    sql postgres 'SELECT a FROM "foo.bar"."c.a.r.t.o.d.b";' should 1
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+
+    sql postgres 'DROP TABLE "foo.bar"."c.a.r.t.o.d.b";'
+    sql postgres 'DROP SCHEMA "foo.bar";'
+}
+
+function test_cdb_querytables_table_name_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE "w.a.d.u.s" (a int);';
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+
+    sql postgres 'DROP TABLE "w.a.d.u.s";';
+}
+
+function test_cdb_querytables_happy_cases() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE wadus (a int);';
+    sql postgres 'CREATE TABLE "FOOBAR" (a int);';
+    sql postgres 'CREATE SCHEMA foo;'
+    sql postgres 'CREATE TABLE foo.wadus (a int);';
+
+    ## See how it does NOT quote anything here
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+
+    ## But it quotes when it's needed even if table name has no dots but was created with quotes
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "FOOBAR"$q$);' should '{"public.\"FOOBAR\""}'
+
+    sql postgres 'DROP TABLE wadus;'
+    sql postgres 'DROP TABLE "FOOBAR";'
+    sql postgres 'DROP TABLE foo.wadus;'
+    sql postgres 'DROP SCHEMA foo;'
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@

test/organization/test.sh

@@ -351,18 +351,43 @@ function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset
 }
 
 function test_cdb_usertables_should_work_with_orgusers() {
-    sql "GRANT USAGE ON SCHEMA cartodb TO publicuser;"
-    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
+    # This test validates the changes proposed in https://github.com/CartoDB/cartodb/pull/5021
+
+    # create tables
     sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
-    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+    sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
     sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
+    
+    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+
+
+    # this is what we need to make public tables available in CDB_UserTables
+    sql postgres "grant publicuser to cdb_testmember_1;"
+    sql postgres "grant publicuser to cdb_testmember_2;"
+
+
+    # this is required to enable select from other schema
+    sql postgres "GRANT USAGE ON SCHEMA cdb_testmember_1 TO publicuser";
+
+
+    # test CDB_UserTables with publicuser
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
     sql publicuser "SELECT count(*) FROM CDB_UserTables('all')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('public')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    sql publicuser "SELECT * FROM CDB_UserTables('all')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('private')" should ""
     # the following tests are for https://github.com/CartoDB/cartodb-postgresql/issues/98
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('all')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('public')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    # cdb_testmember_2 is already owner of `bar` table
+    sql cdb_testmember_2 "select string_agg(t,',') from (select cdb_usertables('all') t order by t) as s" should "bar,test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('private')" should "bar"
+
+    # test cdb_testmember_2 can select from cdb_testmember_1's public table
+    sql cdb_testmember_2 "SELECT * FROM cdb_testmember_1.test_perms_pub" should 1
 
     sql cdb_testmember_1 "DROP TABLE test_perms_pub"
     sql cdb_testmember_1 "DROP TABLE test_perms_priv"