Upgrade version to 0.11.4

174 cartodbfy

.gitignore

@@ -5,4 +5,5 @@ results/
 regression.*
 expected/test
 sql/test
-.idea/*
+.idea/*
+*.swp

CONTRIBUTING.md

@@ -44,17 +44,21 @@ the extension into your test database.
 
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires cheating with PostgreSQL
-so to enforce re-load of the scripts. To help with cheating, "make install"
+as to enforce the scripts to reload. To help with cheating, "make install"
 also installs migration scripts to go from "V" to "V"next and from "V"next
 to "V". Example to upgrade a 0.2.0dev version:
 
 ```sql
-ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
+ALTER EXTENSION cartodb UPDATE TO '0.2.0next';
 ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
 ```
-
 Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
 
 ```sql
 SELECT cartodb.cdb_extension_reload();
 ```
+
+A useful query:
+```sql
+SELECT * FROM pg_extension_update_paths('cartodb') WHERE path IS NOT NULL AND source = cdb_version();
+```

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.8.2
+EXTVERSION = 0.11.4
 
 SED = sed
 
@@ -11,6 +11,7 @@ CDBSCRIPTS = \
   scripts-available/CDB_DDLTriggers.sql \
   scripts-available/CDB_ExtensionPost.sql \
   scripts-available/CDB_ExtensionUtils.sql \
+  scripts-available/CDB_Helper.sql \
   $(END)
 
 UPGRADABLE = \
@@ -41,6 +42,20 @@ UPGRADABLE = \
   0.7.4 \
   0.8.0 \
   0.8.1 \
+  0.8.2 \
+  0.9.0 \
+  0.9.1 \
+  0.9.2 \
+  0.9.3 \
+  0.9.4 \
+  0.10.0 \
+  0.10.1 \
+  0.10.2 \
+  0.11.0 \
+  0.11.1 \
+  0.11.2 \
+  0.11.3 \
+  0.11.4 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,86 @@
+0.11.4 (2015-11-24)
+-------------------
+* Fix for existing PK cartodb_id problem [#174](https://github.com/CartoDB/cartodb-postgresql/issues/174)
+* Add cartodbfication support for column names with embedded points to fix [#6114](https://github.com/CartoDB/cartodb/issues/6114)
+* Disable log invalidation time [#178](https://github.com/CartoDB/cartodb-postgresql/pull/178)
+* Add CDB_GreatCircle for creating great circle routes between two points [#171](https://github.com/CartoDB/cartodb-postgresql/pull/171)
+* Fix to prevent cartodbfication problems [#155](https://github.com/CartoDB/cartodb-postgresql/issues/155)
+
+0.11.3 (2015-10-27)
+-------------------
+* Added CDB_Helper.sql [#173](https://github.com/CartoDB/cartodb-postgresql/pull/173)
+* Added `_CDB_Unique_Identifier` for creating UTF8 aware unique identifiers
+* Added `_CDB_Unique_Column_Identifier` for creating UTF8 aware unique identifiers for columns
+* Added `_CDB_Octet_Truncate` that truncates text to a certain amount of octets.
+
+0.11.2 (2015-10-19)
+-------------------
+* Fix schema not being specified on pg_get_serial_sequence [#170](https://github.com/CartoDB/cartodb-postgresql/pull/170)
+* Log invalidation function call duration in seconds [#163](https://github.com/CartoDB/cartodb-postgresql/pull/163)
+
+0.11.1 (2015-10-06)
+-------------------
+* Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169)
+* cartodbfy now discards cartodb_id candidates that contain nulls [#148](https://github.com/CartoDB/cartodb-postgresql/issues/148)
+
+0.11.0 (2015-09-dd)
+-------------------
+* Groups API
+
+0.10.2 (2015-09-24)
+-------------------
+* Add back the `DROP FUNCTION IF EXISTS CDB_UserTables(text);` to be able to upgrade from `0.7.3` upward [#160](https://github.com/CartoDB/cartodb-postgresql/issues/160)
+
+0.10.1 (2015-09-16)
+-------------------
+* Get back the `update_updated_at` function (still used by old tables) [#143](https://github.com/CartoDB/cartodb-postgresql/pull/143)
+* Fix for CDB_StatsTest.sql test failing randomly [#144](https://github.com/CartoDB/cartodb-postgresql/issues/144)
+* Fix for table cartodbfy'ed without default seq value [#138](https://github.com/CartoDB/cartodb-postgresql/issues/138)
+* Fix for cartodbfy error column `the_geom` already exists [#141](https://github.com/CartoDB/cartodb-postgresql/issues/141)
+* Fix for columns with geometry cartodbfy'ed without SRID [#154](https://github.com/CartoDB/cartodb-postgresql/issues/154)
+
+0.10.0 (2015-09-07)
+-----------------
+* Quote schema and table names returned by CDB_QueryTables [#134](https://github.com/CartoDB/cartodb-postgresql/pull/134). Use quote_ident to quote schema and table names when necessary.
+* Fixed CDB_ColumnNames [#122](https://github.com/CartoDB/cartodb-postgresql/issues/122) and CDB_ColumnType [#130](https://github.com/CartoDB/cartodb-postgresql/issues/130) should honor regclass, returning columns for just the table in the schema and not in any other one [#131](https://github.com/CartoDB/cartodb-postgresql/pull/131).
+* Add kurtosis and skewness [#124](https://github.com/CartoDB/cartodb-postgresql/pull/124).
+* Removed `DROP FUNCTION IF EXISTS cdb_usertables(text);` [#129](https://github.com/CartoDB/cartodb-postgresql/pull/129). This was needed for upgrading between 0.7.4 to 0.8.0 but is no longer needed.
+
+0.9.4 (2015-08-28)
+------------------
+* Fixed issue with indices when renaming tables [#123](https://github.com/CartoDB/cartodb-postgresql/issues/123)
+
+0.9.3 (2015-08-27)
+------------------
+* Modify sampling of quota trigger [#126](https://github.com/CartoDB/cartodb-postgresql/issues/126)
+
+0.9.2 (2015-08-24)
+------------------
+* Fix for `the_geom` column present but not SRID (EWKT) and other corner cases [#121](https://github.com/CartoDB/cartodb-postgresql/pull/121)
+
+0.9.1 (2015-08-19)
+------------------
+* Fix for transformation to webmercator in corner cases [#116](https://github.com/CartoDB/cartodb-postgresql/issues/116)
+
+0.9.0 (2015-08-19)
+------------------
+* Re-implementation of `CDB_CartodbfyTable` functions
+  - The signature of the main function changes to
+    ```
+    FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
+    RETURNS REGCLASS
+    ```
+    - The `destschema` does not need to match the origin schema of `reloid`
+	- It returns the `regclass` of the cartodbfy'ed table, if it needs to be rewritten.
+  - There are many optimizations
+  - The columns `created_at` and `updated_at` will no longer be added
+* Fix for CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` #110
+* Review test to validate permissions in public tables [#112](https://github.com/CartoDB/cartodb-postgresql/pull/112)
+
+0.8.3 (2015-08-14)
+------------------
+* Fixes CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` [#108](https://github.com/CartoDB/cartodb-postgresql/issues/108)
+
 0.8.2 (2015-07-27)
 ------------------
 * Fix for CDB_UserTables returning wrong listings when publicuser is used

README.md

@@ -37,6 +37,8 @@ NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
 
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
+      
+NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
 ---------------
@@ -96,3 +98,8 @@ ls `pg_config --sharedir`/extension/cartodb*
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires special steps documented
 in the CONTRIBUTING document, under "Testing changes live".
+
+Limitations
+-----------
+
+- The main schema of an organization user must have one only owner (the user).

doc/CDB_GreatCircle.md

@@ -0,0 +1,16 @@
+Based on Paul Ramsey's [blog post](http://blog.cartodb.com/jets-and-datelines/).
+#### Using the function
+
+Creates a great circle line.
+
+```sql
+SELECT CDB_GreatCircle(start_point, end_point) FROM table_name
+-- Results a line reprsenting the great circle between the two points
+```
+
+#### Arguments
+
+CDB_GreatCircle(start_point, end_point)
+
+* **start_point** ST_Point indicating the start of the line.
+* **end_point** ST_point indicating the end of the line.

doc/cartodbfy-requirements.rst

@@ -0,0 +1,59 @@
+Introduction
+============
+
+This document aims at describing what cartodbfy is and what its formal requirements are, with the following goals in mind:
+
+- clarify what are the expectations of the "cartodbfycation process".
+- define an important part of what should be a stable, public API
+- allow for better testing, which should in turn...
+- ...ease modifications and increase quality of the code
+
+
+
+What is the cartodbfycation
+===========================
+
+The cartodbfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
+
+
+
+Valid CartoDB tables
+====================
+
+A valid CartoDB table shall meet the following conditions:
+
+- Have a ``cartodb_id`` integer column as primary key with a sequence as default value
+- Have a ``the_geom`` column of type ``Geometry`` with SRID 4326
+- Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857
+- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync
+
+Additionally, a CartoDB table can contain other columns.
+
+
+
+High level requirements
+=======================
+
+Here is a list of high level requirments for the public function ``CDB_CartodbfyTable()``:
+
+- A call to ``CDB_CartodbfyTable()`` shall modify/rewrite the table and produce a valid CartoDB table with the same name.
+- A call to ``CDB_CartodbfyTable()`` shall cause the registration of the table into the platform
+- It shall be idempotent, meaning that successive calls to ``CDB_CartodbfyTable()`` shall not produce any visible effect in the system.
+- If there's a column containing a geometry, it shall be used to generate ``the_geom`` and the ``the_geom_webmercator`` columns.
+- Exporting and re-importing the same table in CartoDB shall produce equivalent tables, with the same features associated to the same ``cartodb_id``'s.
+
+Note that there should be only one feature per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
+
+
+
+Low-level requirements
+======================
+
+- If the original table contains a valid (unique and not null) ``cartodb_id`` column, it shall be used
+- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
+- A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata``
+- A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id``
+- A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator``
+- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns
+
+

expected/test_ddl_triggers.out

@@ -25,6 +25,8 @@ SELECT CDB_SetUserQuotaInBytes('c', 0);
                        0
 (1 row)
 
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
+NOTICE:  role "cartodb_postgresql_unpriv_user" does not exist, skipping
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -36,7 +38,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t3" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
@@ -45,17 +46,6 @@ NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i 
-------------+-----+------+----------+----------------------+---
-          1 | t   | t    |          |                      | 1
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -72,28 +62,14 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as cartodb_id INTO c.t4;
 NOTICE:  trigger "track_updates" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota" for table "c.t4" does not exist, skipping
 NOTICE:  trigger "test_quota_per_row" for table "c.t4" does not exist, skipping
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
-NOTICE:  Trying to recover data from _cartodb_id0 column
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t4) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator 
-------------+-----+------+----------+----------------------
-          1 | t   | t    |          | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -115,27 +91,13 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
+NOTICE:  column "the_geom_webmercator" of relation "t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc 
-------------+-----+------+----------+----------------------+---+---------
-          1 | t   | t    |          |                      | 1 | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -154,27 +116,13 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
+NOTICE:  column "the_geom_webmercator" of relation "t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -196,27 +144,12 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 drop column the_geom_webmercator;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
 NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -239,17 +172,6 @@ select pg_sleep(.1);
 
 alter table c.t3 add column id2 int;
 NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 | id2 
-------------+-----+------+----------+----------------------+---+---------+----------+-----
-          1 | t   | t    |          |                      | 1 |         |          |    
-(1 row)
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -272,5 +194,6 @@ select count(*) from CDB_TableMetadata;
      0
 (1 row)
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

scripts-available/CDB_ColumnNames.sql

@@ -3,11 +3,12 @@ CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
 
-    SELECT column_name 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || '';
+    SELECT c.column_name
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_ColumnType.sql

@@ -3,12 +3,13 @@ CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
 
-    SELECT data_type 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || ''
-        AND column_name = '' || quote_ident($2) || '';
+    SELECT c.data_type
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND column_name = $2
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_Conf.sql

@@ -0,0 +1,48 @@
+----------------------------------
+-- CONF MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by superadmin user.
+-- Functions needing reading configuration should use SECURITY DEFINER.
+----------------------------------
+
+-- This will trigger NOTICE if cartodb.CDB_CONF already exists
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    CREATE TABLE IF NOT EXISTS cartodb.CDB_CONF ( KEY TEXT PRIMARY KEY, VALUE JSON NOT NULL );
+END
+$$;
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.CDB_CONF', '');
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cartodb.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_GetConf(key text)
+    RETURNS JSON AS $$
+DECLARE
+    value JSON;
+BEGIN
+    EXECUTE 'SELECT VALUE FROM cartodb.CDB_CONF WHERE KEY = $1;' INTO value USING key;
+    RETURN value;
+END
+$$ LANGUAGE PLPGSQL STABLE;

scripts-available/CDB_DDLTriggers.sql

@@ -4,17 +4,23 @@ CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
 RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
-  rel_namespace TEXT;
+  rel RECORD;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_relation_create_eventinfo();
 
   -- We're only interested in real relations
   IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid=(event_info.new).relnamespace INTO rel_namespace;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relname::TEXT, rel_namespace, (event_info.new).relnamespace;
+	 event_info.relation, rel.relkind, rel.relname, rel.nspname, rel.relnamespace;
 
   -- We don't want to react to alters triggered by superuser,
   IF current_setting('is_superuser') = 'on' THEN
@@ -25,15 +31,15 @@ BEGIN
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
   -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   RAISE DEBUG 'Inserting into cartodb.CDB_TableMetadata';
 
   -- Add entry to CDB_TableMetadata (should CartodbfyTable do this?)
-  INSERT INTO cartodb.CDB_TableMetadata(tabname,updated_at) 
-  VALUES (event_info.relation, now());
+  INSERT INTO cartodb.CDB_TableMetadata(tabname, updated_at) 
+  VALUES (newtable, now());
 
 END; $$;
 -- }
@@ -66,14 +72,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
-  rel_namespace TEXT;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_alter_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % altered by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -84,16 +95,14 @@ BEGIN
     RETURN;
   END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
-
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
   WHERE tabname = event_info.relation; 
 
 END; $$;
@@ -106,14 +115,19 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
-  rel_namespace TEXT;
+  newtable REGCLASS;
 BEGIN
   event_info := schema_triggers.get_column_drop_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % drop by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.old).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;
@@ -124,17 +138,16 @@ BEGIN
     RETURN;
   END IF;
 
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
 
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
+  newtable := cartodb.CDB_CartodbfyTable(rel.nspname, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
   -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
+  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW(), tabname = newtable
+  WHERE tabname = event_info.relation;
 
 END; $$;
 -- }
@@ -149,10 +162,15 @@ DECLARE
 BEGIN
   event_info := schema_triggers.get_column_add_eventinfo();
 
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
+  SELECT c.relname, c.relnamespace, c.relkind, n.nspname 
+  FROM pg_class c
+  JOIN pg_namespace n 
+  ON c.relnamespace = n.oid
+  WHERE c.oid = event_info.relation
+  INTO rel;
 
   RAISE DEBUG 'Column % added by % (superuser? %) in relation % of kind %',
-	 (event_info.new).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
+	 (event_info.new).attname, current_user, current_setting('is_superuser'), rel.relname, rel.relkind;
 
   -- We're only interested in real relations
   IF rel.relkind != 'r' THEN RETURN; END IF;

scripts-available/CDB_DateToNumber.sql

@@ -13,3 +13,19 @@ RETURN output;
 END;
 $$
 LANGUAGE 'plpgsql' STABLE STRICT;
+
+-- Convert timestamp with time zone to double precision
+--
+CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp with time zone)
+RETURNS double precision AS $$
+DECLARE output double precision;
+BEGIN
+    BEGIN
+        SELECT extract (EPOCH FROM input) INTO output;
+    EXCEPTION WHEN OTHERS THEN
+        RETURN NULL;
+    END;
+RETURN output;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE STRICT;

scripts-available/CDB_GreatCircle.sql

@@ -0,0 +1,26 @@
+-- Great circle point-to-point routes, based on:
+--   http://blog.cartodb.com/jets-and-datelines/
+--
+CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry, max_segment_length NUMERIC DEFAULT 100000)
+RETURNS geometry AS $$
+DECLARE
+  line geometry;
+BEGIN
+  line = ST_Segmentize(
+    ST_Makeline(
+      start_point,
+      end_point
+    )::geography,
+    max_segment_length
+  )::geometry;
+
+  IF ST_XMax(line) - ST_XMin(line) > 180 THEN
+    line = ST_Difference(
+      ST_Shift_Longitude(line),
+			ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001)
+		);
+  END IF;
+RETURN line;
+END;
+$$
+LANGUAGE 'plpgsql';

scripts-available/CDB_Groups.sql

@@ -0,0 +1,252 @@
+----------------------------------
+-- GROUP MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by org admin. See CDB_Organization_AddAdmin.
+----------------------------------
+
+-- Creates a new group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_CreateGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('CREATE ROLE %I NOLOGIN;', group_role);
+    PERFORM cartodb._CDB_Group_CreateGroup_API(group_name, group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Drops group and everything that role owns
+-- TODO: LIMITATION: in order to drop a role all its owned objects must be dropped before.
+-- Right now this is done with DROP OWNED, which can only be done by a superadmin.
+-- Not even the role creator can drop the role and the objects it owns.
+-- All group owned objects by the group are permissions.
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_DropGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('DROP OWNED BY %I', group_role);
+    EXECUTE format('DROP ROLE IF EXISTS %I', group_role);
+    PERFORM cartodb._CDB_Group_DropGroup_API(group_name);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Renames a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RenameGroup(old_group_name text, new_group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    old_group_role TEXT;
+    new_group_role TEXT;
+BEGIN
+    old_group_role = cartodb._CDB_Group_GroupRole(old_group_name);
+    new_group_role = cartodb._CDB_Group_GroupRole(new_group_name);
+    EXECUTE format('ALTER ROLE %I RENAME TO %I', old_group_role, new_group_role);
+    PERFORM cartodb._CDB_Group_RenameGroup_API(old_group_name, new_group_name, new_group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Adds users to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_AddUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      IF(group_role IS NULL OR user_role IS NULL)
+      THEN
+        RAISE EXCEPTION 'Group role (%) and user role (%) must be already existing', group_role, user_role;
+      END IF;
+      EXECUTE format('GRANT %I TO %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_AddUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Removes users from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RemoveUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      EXECUTE format('REVOKE %I FROM %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_RemoveUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+----------------------------------
+-- TABLE MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by table owners.
+----------------------------------
+
+-- Grants table read permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantRead(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantRead(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantRead(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT ON TABLE %I.%I TO %I', username, table_name, group_role );
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'r');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Grants table write permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantReadWrite(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE %I.%I TO %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, true);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'w');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Granting and revoking permissions on sequences
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_TableSequences_Permission(group_name text, username text, table_name text, do_grant bool)
+    RETURNS VOID AS $$
+DECLARE
+    column_name TEXT;
+    sequence_name TEXT;
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    FOR column_name IN EXECUTE 'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = current_database() AND TABLE_SCHEMA = $1 AND TABLE_NAME = $2 AND COLUMN_DEFAULT LIKE ''nextval%''' USING username, table_name
+    LOOP
+        EXECUTE format('SELECT PG_GET_SERIAL_SEQUENCE(''%I.%I'', ''%I'')', username, table_name, column_name) INTO sequence_name;
+        IF sequence_name IS NOT NULL THEN
+          IF do_grant THEN
+            -- Here %s is needed since sequence_name has quotes
+            EXECUTE format('GRANT USAGE, SELECT, UPDATE ON SEQUENCE %s TO %I', sequence_name, group_role);
+          ELSE
+            EXECUTE format('REVOKE ALL ON SEQUENCE %s FROM %I', sequence_name, group_role);
+          END IF;
+        END IF;
+    END LOOP;
+    RETURN;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Revokes all permissions on a table from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_RevokeAll(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_RevokeAll(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAll(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('REVOKE ALL ON TABLE %I.%I FROM %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, false);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name, username, table_name);
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-----------------------
+-- Helper functions
+-----------------------
+-- Given a group name returns a role. group_name must be a valid PostgreSQL idenfifier. See http://www.postgresql.org/docs/9.2/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_GroupRole(group_name text)
+    RETURNS TEXT AS $$
+DECLARE
+    group_role TEXT;
+    prefix TEXT;
+    max_length constant INTEGER := 63;
+BEGIN
+    prefix = format('%s_g_', cartodb._CDB_Group_ShortDatabaseName());
+    group_role := format('%s%s', prefix, group_name);
+    IF LENGTH(group_role) > max_length
+    THEN
+        RAISE EXCEPTION 'Group name must be shorter. It can''t have more than % characters, but it is longer (%): %', max_length - LENGTH(prefix), length(group_name), group_name;
+    END IF;
+    RETURN group_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Returns the first owner of the schema matching username. Organization user schemas must have one only owner.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_User_RoleFromUsername(username text)
+    RETURNS TEXT AS $$
+DECLARE
+    user_role TEXT;
+BEGIN
+    -- This was preferred, but non-superadmins won't get results
+    -- SELECT SCHEMA_OWNER FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = $1 LIMIT 1'
+    SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = username INTO user_role;
+    RETURN user_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Database names are too long, we need a shorter version for composing role names
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_ShortDatabaseName()
+    RETURNS TEXT AS $$
+DECLARE
+    short_database_name TEXT;
+BEGIN
+    SELECT md5(current_database()) INTO short_database_name;
+    RETURN short_database_name;
+END
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_Groups_API.sql

@@ -0,0 +1,195 @@
+----------------------------------
+-- GROUP METADATA API FUNCTIONS
+--
+-- Meant to be used by CDB_Group_* functions to sync data with the editor.
+-- Requires configuration parameter. Example: SELECT cartodb.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
+----------------------------------
+
+-- TODO: delete this development cleanup before final merge
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_AddMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_RemoveMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_AddMember_API(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_RemoveMember_API(group_name text, username text);
+
+-- Sends the create group request
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_CreateGroup_API(group_name text, group_role text)
+    RETURNS VOID AS
+$$
+    import string
+
+    url = '/api/v1/databases/{0}/groups'
+    body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_DropGroup_API(group_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_AddUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantPermission_API(group_name text, username text, table_name text, access text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    body = '{ "access": "%s" }' % access
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name text, username text, table_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE TYPE _CDB_Group_API_Params AS (
+    host text,
+    port int,
+    timeout int,
+    auth text
+);
+
+-- This must be explicitally extracted because "composite types are currently not supported".
+-- See http://www.postgresql.org/docs/9.3/static/plpython-database.html.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Conf()
+    RETURNS _CDB_Group_API_Params AS
+$$
+    conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('groups_api') conf")[0]['conf']
+    if conf is None:
+      return None
+    else:
+      import json
+      params = json.loads(conf)
+      auth = 'Basic %s' % plpy.execute("SELECT cartodb._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
+      return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Auth(username text, password text)
+    RETURNS TEXT AS
+$$
+    import base64
+    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+-- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
+    RETURNS int AS
+$$
+    import httplib
+
+    params = plpy.execute("select c.host, c.port, c.timeout, c.auth from cartodb._CDB_Group_API_Conf() c;")[0]
+    if params['host'] is None:
+      return None
+
+    headers = { 'Authorization': params['auth'], 'Content-Type': 'application/json', 'X-Forwarded-Proto': 'https' }
+
+    retry = 3
+
+    last_err = None
+    while retry > 0:
+      try:
+        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        database_name = plpy.execute("select current_database();")[0]['current_database']
+        client.request(method, url.format(database_name), body, headers)
+        response = client.getresponse()
+        assert response.status in valid_return_codes
+        return response.status
+      except Exception as err:
+        retry -= 1
+        last_err = err
+        plpy.warning('Retrying after: ' + str(err))
+        client = SD['groups_api_client'] = None
+
+    if last_err is not None:
+      plpy.error('Fatal Group API error: ' + str(last_err))
+      raise last_err
+
+    return None
+$$ LANGUAGE 'plpythonu' VOLATILE;
+revoke all on function cartodb._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_Helper.sql

@@ -0,0 +1,156 @@
+-- UTF8 safe and length aware. Find a unique identifier with a given prefix
+-- and/or suffix and withing a schema. If a schema is not specified, the identifier
+-- is guaranteed to be unique for all schemas.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)
+RETURNS TEXT
+AS $$
+DECLARE
+  maxlen CONSTANT INTEGER := 63;
+
+  rec RECORD;
+  usedspace INTEGER;
+  ident TEXT;
+  origident TEXT;
+  candrelname TEXT;
+
+  i INTEGER;
+BEGIN
+  -- Accounts for the _XX incremental suffix in case the identifier is taken
+  usedspace := 3;
+  usedspace := usedspace + coalesce(octet_length(prefix), 0);
+  usedspace := usedspace + coalesce(octet_length(suffix), 0);
+
+  candrelname := _CDB_Octet_Truncate(relname, maxlen - usedspace);
+
+  IF candrelname = '' THEN
+    PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier');
+  END IF;
+
+  ident := coalesce(prefix, '') || candrelname || coalesce(suffix, '');
+
+  i := 0;
+  origident := ident;
+
+  WHILE i < 100 LOOP
+    IF schema IS NOT NULL THEN
+      SELECT c.relname, n.nspname
+      INTO rec
+      FROM pg_class c
+      JOIN pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.relname = ident
+      AND n.nspname = schema;
+    ELSE
+      SELECT c.relname, n.nspname
+      INTO rec
+      FROM pg_class c
+      JOIN pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.relname = ident;
+    END IF;
+
+    IF NOT FOUND THEN
+      RETURN ident;
+    END IF;
+
+    ident := origident || '_' || i;
+    i := i + 1;
+  END LOOP;
+
+  PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier');
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- UTF8 safe and length aware. Find a unique identifier for a column with a given prefix
+-- and/or suffix based on colname and within a relation specified via reloid.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, colname TEXT, suffix TEXT, reloid REGCLASS)
+RETURNS TEXT
+AS $$
+DECLARE
+  maxlen CONSTANT INTEGER := 63;
+
+  rec RECORD;
+  candcolname TEXT;
+  usedspace INTEGER;
+  ident TEXT;
+  origident TEXT;
+
+  i INTEGER;
+BEGIN
+  -- Accounts for the _XX incremental suffix in case the identifier is taken
+  usedspace := 3;
+  usedspace := usedspace + coalesce(octet_length(prefix), 0);
+  usedspace := usedspace + coalesce(octet_length(suffix), 0);
+
+  candcolname := _CDB_Octet_Truncate(colname, maxlen - usedspace);
+
+  IF candcolname = '' THEN
+    PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier');
+  END IF;
+
+  ident := coalesce(prefix, '') || candcolname || coalesce(suffix, '');
+
+  i := 0;
+  origident := ident;
+
+  WHILE i < 100 LOOP
+    SELECT a.attname
+    INTO rec
+    FROM pg_class c
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE NOT a.attisdropped
+    AND a.attnum > 0
+    AND c.oid = reloid
+    AND a.attname = ident;
+
+    IF NOT FOUND THEN
+      RETURN ident;
+    END IF;
+
+    ident := origident || '_' || i;
+    i := i + 1;
+  END LOOP;
+
+  PERFORM _CDB_Error('looping too far', '_CDB_Unique_Column_Identifier');
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- Truncates a given string to a max_octets octets taking care
+-- not to leave characters in half. UTF8 safe.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Truncate(string TEXT, max_octets INTEGER)
+RETURNS TEXT
+AS $$
+DECLARE
+  extcharlen CONSTANT INTEGER := octet_length('ñ');
+
+  expected INTEGER;
+  examined INTEGER;
+  strlen INTEGER;
+
+  i INTEGER;
+BEGIN
+
+  IF max_octets <= 0 THEN
+    RETURN '';
+  ELSIF max_octets >= octet_length(string) THEN
+    RETURN string;
+  END IF;
+
+  strlen := char_length(string);
+
+  expected := char_length(string);
+  examined := octet_length(string);
+
+  IF expected = examined THEN
+    RETURN left(string, max_octets);
+  END IF;
+
+  i := max_octets / extcharlen;
+
+  WHILE octet_length(left(string, i)) <= max_octets LOOP
+    i := i + 1;
+  END LOOP;
+
+  RETURN left(string, (i - 1));
+END;
+$$ LANGUAGE 'plpgsql';

scripts-available/CDB_Organizations.sql

@@ -1,14 +1,16 @@
 CREATE OR REPLACE
 FUNCTION cartodb.CDB_Organization_Member_Group_Role_Member_Name()
     RETURNS TEXT
-AS 'SELECT ''cdb_org_member''::text || ''_'' || md5(current_database());'
+AS $$
+    SELECT 'cdb_org_member'::text || '_' || md5(current_database());
+$$
 LANGUAGE SQL IMMUTABLE;
 
 DO LANGUAGE 'plpgsql' $$
 DECLARE
     cdb_org_member_role_name TEXT;
 BEGIN
-    cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
+  cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
   IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_member_role_name )
   THEN
     EXECUTE 'CREATE ROLE "' || cdb_org_member_role_name || '" NOLOGIN;';
@@ -25,6 +27,60 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
+-------------------------------------------------------------------------------
+-- Administrator
+-------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Organization_Admin_Role_Name()
+    RETURNS TEXT
+AS $$
+    SELECT current_database() || '_a'::text;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+-- Administrator role creation on extension install
+DO LANGUAGE 'plpgsql' $$
+DECLARE
+    cdb_org_admin_role_name TEXT;
+BEGIN
+    cdb_org_admin_role_name := cartodb._CDB_Organization_Admin_Role_Name();
+    IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_admin_role_name )
+    THEN
+        EXECUTE format('CREATE ROLE %I CREATEROLE NOLOGIN;', cdb_org_admin_role_name);
+    END IF;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_AddAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('GRANT %I TO %I WITH ADMIN OPTION', cdb_admin_role, cdb_user_role);
+    -- CREATEROLE is not inherited, and is needed for user creation
+    EXECUTE format('ALTER ROLE %I CREATEROLE', cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_RemoveAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('ALTER ROLE %I NOCREATEROLE', cdb_user_role);
+    EXECUTE format('REVOKE %I FROM %I', cdb_admin_role, cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
 
 -------------------------------------------------------------------------------
 -- Sharing tables

scripts-available/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;

scripts-available/CDB_Quota.sql

@@ -1,3 +1,19 @@
+CREATE OR REPLACE FUNCTION cartodb._CDB_total_relation_size(_schema_name TEXT, _table_name TEXT)
+RETURNS bigint AS
+$$
+DECLARE relation_size bigint := 0;
+BEGIN
+  BEGIN
+    SELECT pg_total_relation_size(format('"%s"."%s"', _schema_name, _table_name)) INTO relation_size;
+  EXCEPTION
+    WHEN undefined_table OR OTHERS THEN
+      RAISE NOTICE 'cartodb._CDB_total_relation_size(''%'', ''%'') caught error: % (%)', _schema_name, _table_name, SQLERRM, SQLSTATE;
+  END;
+  RETURN relation_size;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;
+
 -- Return the estimated size of user data. Used for quota checking.
 CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
 RETURNS bigint AS
@@ -24,7 +40,7 @@ BEGIN
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(pg_total_relation_size('"' || schema_name || '"."' || table_name || '"')))) table_size,
+    SELECT COALESCE(INT8(SUM(cartodb._CDB_total_relation_size(schema_name, table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1

scripts-available/CDB_Stats.sql

@@ -0,0 +1,47 @@
+--
+-- Calculate basic statistics of a given dataset
+--
+-- @param in_array A numeric array of numbers
+--
+-- Returns: statistical quantity chosen
+-- 
+-- References: http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm
+--
+
+-- Calculate kurtosis
+CREATE OR REPLACE FUNCTION CDB_Kurtosis ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    k numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 4)) / ( $2 * power($3, 4)) - 3
+             FROM (SELECT unnest($4) e ) x'
+    INTO k
+    USING a, c, s, in_array;
+
+    RETURN k;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Calculate skewness
+CREATE OR REPLACE FUNCTION CDB_Skewness ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    sk numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 * power($3, 3))
+             FROM (SELECT unnest($4) e ) x'
+    INTO sk
+    USING a, c, s, in_array;
+
+    RETURN sk;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_TableMetadata.sql

@@ -64,6 +64,10 @@ DECLARE
   tabname TEXT;
   rec RECORD;
   found BOOL;
+  function_start timestamptz;
+  function_end timestamptz;
+  function_duration float;
+  log_error_verbosity_value text;
 BEGIN
 
   IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
@@ -72,6 +76,8 @@ BEGIN
     tabname = OLD.tabname;
   END IF;
 
+  function_start := clock_timestamp();
+ 
   -- Notify table data update
   -- This needs a little bit more of research regarding security issues
   -- see https://github.com/CartoDB/cartodb/pull/241
@@ -105,7 +111,14 @@ BEGIN
     EXIT;
   END LOOP;
   IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF;
-
+  
+  function_end := clock_timestamp();
+  SELECT extract(epoch from (function_end - function_start)) INTO function_duration;
+  SELECT setting INTO log_error_verbosity_value FROM pg_settings WHERE name='log_error_verbosity';
+  SET log_error_verbosity=TERSE;
+  RAISE LOG 'invalidation_duration: %', function_duration::text;
+  PERFORM 'SET log_error_verbosity= ' || log_error_verbosity_value;
+  
   RETURN NULL;
 END;
 $$

scripts-available/CDB_UserTables.sql

@@ -5,7 +5,7 @@
 --
 -- Currently accepted permissions are: 'public', 'private' or 'all'
 --
-DROP FUNCTION IF EXISTS cdb_usertables(text);
+DROP FUNCTION IF EXISTS CDB_UserTables(text);
 CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
 RETURNS SETOF name
 AS $$
@@ -15,7 +15,7 @@ FROM pg_class c
 JOIN pg_namespace n ON n.oid = c.relnamespace
 WHERE c.relkind = 'r' 
 AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology')
+AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
 AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')

scripts-enabled/241-CDB_GreatCircle.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_GreatCircle.sql

scripts-enabled/CDB_Conf.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Conf.sql

scripts-enabled/CDB_Groups.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups.sql

scripts-enabled/CDB_Groups_API.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups_API.sql

scripts-enabled/CDB_Stats.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Stats.sql

sql/test_ddl_triggers.sql

@@ -10,6 +10,7 @@ create schema c;
 
 SELECT CDB_SetUserQuotaInBytes('c', 0);
 
+DROP USER IF EXISTS cartodb_postgresql_unpriv_user;
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -21,13 +22,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select 1 as i INTO c.t3;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -38,11 +32,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 -- Table with cartodb_id field, see
 -- http://github.com/CartoDB/cartodb-postgresql/issues/32
 select 1 as cartodb_id INTO c.t4;
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
 
 RESET SESSION AUTHORIZATION;
 select
@@ -58,13 +47,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -75,13 +57,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -95,13 +70,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 drop column the_geom_webmercator;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -115,13 +83,6 @@ SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
 select pg_sleep(.1);
 alter table c.t3 add column id2 int;
 
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
-
 RESET SESSION AUTHORIZATION;
 select
  tabname::text,
@@ -136,5 +97,6 @@ RESET SESSION AUTHORIZATION;
 drop schema c cascade;
 select count(*) from CDB_TableMetadata;
 
-DROP USER cartodb_postgresql_unpriv_user;
+DROP OWNED BY cartodb_postgresql_unpriv_user;
+DROP ROLE cartodb_postgresql_unpriv_user;
 DROP FUNCTION _CDB_UserQuotaInBytes();

sql/test_setup.sql

@@ -7,4 +7,4 @@ RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
-$$ LANGUAGE 'plpgsql';
+$$ LANGUAGE 'plpgsql';

test/CDB_CartodbfyTableTest.sql

@@ -12,11 +12,12 @@ DECLARE
   tmp INTEGER;
   ogc_geom geometry_columns; -- old the_geom record in geometry_columns
   ogc_merc geometry_columns; -- old the_geom_webmercator record in geometry_columns
+  tabtext TEXT;
 BEGIN
 
   -- Save current constraints on geometry columns, if any
-  ogc_geom = ('','','','',0,0,'GEOMETRY'); 
-  ogc_merc = ogc_geom; 
+  ogc_geom = ('','','','',0,0,'GEOMETRY');
+  ogc_merc = ogc_geom;
   sql := 'SELECT gc.* FROM geometry_columns gc, pg_class c, pg_namespace n '
     || 'WHERE c.oid = ' || tabname::oid || ' AND n.oid = c.relnamespace'
     || ' AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname'
@@ -30,24 +31,17 @@ BEGIN
     END IF;
   END LOOP;
 
+  tabtext := Format('%s.%s','public',tabname);
+  RAISE NOTICE 'CARTODBFYING % !!!!', tabtext;
   PERFORM CDB_CartodbfyTable('public', tabname);
+  tabname := tabtext::regclass;
 
   sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
   EXECUTE sql INTO STRICT id;
-  sql := 'SELECT created_at,updated_at,the_geom_webmercator FROM '
+  sql := 'SELECT the_geom_webmercator FROM '
     || tabname::text || ' WHERE cartodb_id = ' || id;
   EXECUTE sql INTO STRICT rec;
 
-  -- Check created_at and updated_at at creation time
-  lag = rec.created_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'created_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-  lag = rec.updated_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'updated_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-
   -- Check the_geom_webmercator trigger
   IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
     RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
@@ -72,7 +66,7 @@ BEGIN
            ELSE 3857 END as expsrid,
       CASE WHEN gc.f_geometry_column = 'the_geom' THEN ogc_geom.type
            ELSE ogc_merc.type END as exptype, gc.*
-    FROM geometry_columns gc, pg_class c, pg_namespace n 
+    FROM geometry_columns gc, pg_class c, pg_namespace n
     WHERE c.oid = tabname::oid AND n.oid = c.relnamespace
           AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname
           AND gc.f_geometry_column IN ( 'the_geom', 'the_geom_webmercator')
@@ -84,7 +78,7 @@ BEGIN
         rec.f_geometry_column, rec.srid, rec.expsrid;
     END IF;
     -- Check TYPE constraint didn't change
-    IF rec.type != rec.exptype THEN
+    IF (rec.type != 'GEOMETRY') AND (rec.type != 'POINT') THEN
       RAISE EXCEPTION 'type of % in geometry_columns is %, expected %',
         rec.f_geometry_column, rec.type, rec.exptype;
     END IF;
@@ -94,7 +88,7 @@ BEGIN
       RAISE EXCEPTION '% entries found for table % in geometry_columns, expected 2', tmp, tabname;
   END IF;
 
-  -- Check GiST index 
+  -- Check GiST index
   sql := 'SELECT a.attname, count(ri.relname) FROM'
     || ' pg_index i, pg_class c, pg_class ri, pg_attribute a, pg_opclass o'
     || ' WHERE i.indrelid = c.oid AND ri.oid = i.indexrelid'
@@ -115,10 +109,10 @@ BEGIN
   -- Check null constraint on cartodb_id, created_at, updated_at
   SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
     AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
-      ( 'cartodb_id', 'created_at', 'updated_at' )
+      ( 'cartodb_id' )
     AND NOT a.attnotnull INTO strict tmp;
   IF tmp > 0 THEN
-      RAISE EXCEPTION 'cartodb_id or created_at or updated_at are missing not-null constraint';
+      RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
   END IF;
 
   -- Cleanup
@@ -170,34 +164,48 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- table with existing updated_at and created_at fields ot type text
-CREATE TABLE t AS SELECT NOW()::text as created_at,
-                         NOW()::text as updated_at,
-                         NOW() as reftime;
-SELECT CDB_CartodbfyTableCheck('t', 'text timestamps');
-SELECT extract(secs from reftime-created_at),
-       extract(secs from reftime-updated_at) FROM t;
-CREATE VIEW v AS SELECT * FROM t;
-SELECT CDB_CartodbfyTableCheck('t', 'cartodbfied with view');
-DROP VIEW v;
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text
+-- CREATE TABLE t AS SELECT 10::text as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+-- select cartodb_id/2 FROM t;
+-- DROP TABLE t;
 
--- table with existing cartodb_id field of type text
-CREATE TABLE t AS SELECT 10::text as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
-select cartodb_id/2 FROM t; 
-DROP TABLE t;
-
--- table with existing cartodb_id field of type text not casting
-CREATE TABLE t AS SELECT 'nan' as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
-select cartodb_id,_cartodb_id0 FROM t; 
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text not casting
+-- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+-- select cartodb_id,_cartodb_id0 FROM t;
+-- DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
 SELECT CDB_CartodbfyTableCheck('t', 'unsequenced cartodb_id');
-select cartodb_id FROM t; 
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column
+CREATE TABLE t AS SELECT 'SRID=4326;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, no SRS
+CREATE TABLE t AS SELECT 'POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, no srs');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, unusual SRS
+CREATE TABLE t AS SELECT 'SRID=26910;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, srs = 26819');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text unparseable geometry column
+CREATE TABLE t AS SELECT 'SRID=26910;PONT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, unparseable content');
+SELECT cartodb_id FROM t;
 DROP TABLE t;
 
 -- table with existing cartodb_id serial primary key
@@ -208,21 +216,150 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
 AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
 DROP TABLE t;
 
--- table with existing the_geom and created_at and containing null values
--- Really, a test for surviving an longstanding PostgreSQL bug:
--- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
-CREATE TABLE t (
-    the_geom geometry(Geometry,4326),
-    created_at timestamptz,
-    updated_at timestamptz
+-- tables can be renamed and there's no index name clashing #123
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+ALTER TABLE original RENAME TO original_renamed;
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+DROP TABLE original_renamed;
+DROP TABLE original;
+
+-- Table always have a default seq value after cartodbfy #138
+CREATE TABLE bug_empty_table_no_seq (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
 );
-COPY t (the_geom, created_at, updated_at) FROM stdin;
-0106000020E610000001000000010300000001000000050000009EB8244146435BC017B65E062AD343409EB8244146435BC0F51AF6E2708044400B99891683765AC0F51AF6E2708044400B99891683765AC017B65E062AD343409EB8244146435BC017B65E062AD34340	2012-06-06 21:59:08	2013-06-10 20:17:20
-0106000020E61000000100000001030000000100000005000000DA7763431A1A5CC0FBCEE869313C3A40DA7763431A1A5CC09C1B8F55BC494440F9F4A9C7993356C09C1B8F55BC494440F9F4A9C7993356C0FBCEE869313C3A40DA7763431A1A5CC0FBCEE869313C3A40	2012-06-06 21:59:08	2013-06-10 20:17:20
-\N	\N	\N
-\.
-SELECT CDB_CartodbfyTableCheck('t', 'null geom and timestamp values');
-DROP TABLE t;
+SELECT CDB_CartodbfyTableCheck('bug_empty_table_no_seq', 'Table always have a default seq value after cartodbfy #138');
+INSERT INTO bug_empty_table_no_seq DEFAULT VALUES;
+DROP TABLE bug_empty_table_no_seq;
+
+-- Existing cartodb_id values are respected
+CREATE table existing_cartodb_id (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+INSERT INTO existing_cartodb_id (cartodb_id, description) VALUES
+       (10, 'a'),
+       (20, 'b'),
+       (30, 'c');
+SELECT CDB_CartodbfyTableCheck('existing_cartodb_id', 'Existing cartodb_id values are respected #138');
+SELECT cartodb_id,the_geom,the_geom_webmercator,description,name from existing_cartodb_id;
+DROP TABLE existing_cartodb_id;
+
+-- Table with both the_geom and wkb_geometry
+CREATE TABLE many_geometry_columns (
+       the_geom geometry,
+       wkb_geometry geometry(MultiPoint,4326),
+       description varchar
+);
+INSERT INTO many_geometry_columns (the_geom, wkb_geometry) VALUES
+       ('0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440', '0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440'),
+       ('0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440', '0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440');
+SELECT CDB_CartodbfyTableCheck('many_geometry_columns', 'Table with both the_geom and wkb_geometry #141');
+SELECT * FROM many_geometry_columns;
+DROP TABLE many_geometry_columns;
+
+-- Many colliding geom columns
+CREATE TABLE many_colliding_columns (
+  the_geom varchar,
+  the_geom_webmercator varchar,
+  my_geom geometry,
+  my_mercgeom geometry(Point, 3857),
+  cartodb_id varchar,
+  my_pk integer primary key
+);
+INSERT INTO many_colliding_columns VALUES (
+ 'foo', 'bar', 'SRID=4326;POINT(0 0)', 'SRID=3857;POINT(0 0)', 'nerf', 1
+);
+SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141');
+DROP TABLE many_colliding_columns;
+
+-- Table with null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+-- Table with non unique cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+-- Table with non unique and null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  cartodb_id integer
+);
+CREATE UNIQUE INDEX "test_cartodb_id_key" ON test (cartodb_id);
+CREATE UNIQUE INDEX "test_cartodb_id_pkey" ON test (cartodb_id);
+ALTER TABLE test ADD CONSTRAINT "test_pkey" PRIMARY KEY USING INDEX test_cartodb_id_pkey;
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with primary key and unique index on it #174');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  name varchar,
+  "first.value" integer,
+  "second.value" integer
+);
+INSERT INTO test VALUES ('one', 1, 2), ('two', 3, 4);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with dots in name columns (cartodb #6114)');
+SELECT name, "first.value" from test;
+DROP TABLE test;
+
+SET client_min_messages TO notice;
+-- _CDB_create_cartodb_id_column with cartodb_id integer already present
+CREATE TABLE test (cartodb_id integer);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+
+-- _CDB_create_cartodb_id_column with cartodb_id text already present
+CREATE TABLE test (cartodb_id text);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+SET client_min_messages TO error;
 
 -- TODO: table with existing custom-triggered the_geom
 

test/CDB_CartodbfyTableTest_expect

@@ -2,8 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 7 at PERFORM
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
@@ -28,30 +28,109 @@ trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
 DROP TABLE
 SELECT 1
-text timestamps cartodbfied fine
-0|0
-CREATE VIEW
-cartodbfied with view cartodbfied fine
-DROP VIEW
-DROP TABLE
-SELECT 1
-text cartodb_id cartodbfied fine
-5
-DROP TABLE
-SELECT 1
-uncasting text cartodb_id cartodbfied fine
-1|nan
-DROP TABLE
-SELECT 1
 unsequenced cartodb_id cartodbfied fine
 1
 DROP TABLE
+SELECT 1
+text the_geom column cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, no srs cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, srs = 26819 cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, unparseable content cartodbfied fine
+1
+DROP TABLE
 CREATE TABLE
 cartodb_id serial primary key cartodbfied fine
 t_pkey|cartodb_id
 DROP TABLE
 CREATE TABLE
-null geom and timestamp values cartodbfied fine
+original
+ALTER TABLE
+CREATE TABLE
+original
 DROP TABLE
+DROP TABLE
+CREATE TABLE
+Table always have a default seq value after cartodbfy #138 cartodbfied fine
+INSERT 0 1
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Existing cartodb_id values are respected #138 cartodbfied fine
+10|||a|
+20|||b|
+30|||c|
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with both the_geom and wkb_geometry #141 cartodbfied fine
+1|0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440|0104000020110F00000100000001010000004A9F662B456D5FC11392690DC3F75241|
+2|0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440|0104000020110F00000100000001010000002858E0EC376D5FC1CAE8DB4B95F55241|
+DROP TABLE
+CREATE TABLE
+INSERT 0 1
+Many colliding columns #141 cartodbfied fine
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|3
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Table with non unique cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|2
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with non unique and null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|2
+DROP TABLE
+CREATE TABLE
+CREATE INDEX
+CREATE INDEX
+ALTER TABLE
+INSERT 0 3
+Table with primary key and unique index on it #174 cartodbfied fine
+1
+2
+3
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with dots in name columns (cartodb #6114) cartodbfied fine
+one|1
+two|3
+DROP TABLE
+SET
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+SET
 DROP FUNCTION
 DROP FUNCTION

test/CDB_DateToNumberTest.sql

@@ -0,0 +1,2 @@
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00'::timestamp);
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+05'::timestamp with time zone);

test/CDB_DateToNumberTest_expect

@@ -0,0 +1,2 @@
+915753600
+915735600

test/CDB_GreatCircle.sql

@@ -0,0 +1,2 @@
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)));
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000));

test/CDB_GreatCircle_expect

@@ -0,0 +1,2 @@
+LINESTRING(4.259 55.858,5.6692453115051 56.0150275120673,7.10720375678704 56.157400475677,8.5718366560563 56.2842986378254,10.0619272412891 56.3949153508462,11.5760785994189 56.4884642014437,13.1127142001617 56.564185934303,14.6700812655504 56.6213555706215,16.2462571744128 56.6592896061102,17.8391590143095 56.6773531596105,19.4465562981665 56.6749669334121,21.0660867567155 56.6516138405427,22.6952750058883 56.6068451534252,24.3315537765309 56.540286032869,25.9722872888145 56.4516403065472,27.6147962622065 56.3406943817481,29.2563839799455 56.207320197769,30.8943627796619 56.0514771479657,32.5260803224591 55.8732129290618,34.1489450028345 55.6726633044968,35.7604499005266 55.4500507979281,37.3581947399686 55.2056823610616,38.9399054089486 54.9399460854786,40.5034506895044 54.6533070499613,42.0468559644411 54.3463024122038,43.5683137754523 54.0195358662507,45.066191217402 53.673671594382,46.5390342525062 53.3094278446298,47.9855691138079 52.9275702630659,49.4047010366934 52.5289051040742,50.7955106088955 52.1142724327331,52.1572480633875 51.6845394219955,53.4893258557794 51.2405938343407,54.7913098701049 50.7833377637432,56.0629095865715 50.3136816997865,57.3039675245588 49.8325389621027,58.5144482465496 49.3408205404538,59.6944271762829 48.8394303639846,60.8440794494795 48.329261012675,61.9636689799149 47.811189874886,63.0535378889196 47.2860757471582,64.1140964137264 46.7547558660385,65.1458133802427 46.2180433565989,66.1492072992903 45.676725078361,67.1248381223566 45.131559846414,68.0732996734468 44.583277003498,68.9952127576034 44.0325753175597,69.8912189338183 43.4801221786776,70.7619749300985 42.9265530691612,71.6081476710291 42.3724712809595,72.4304098829428 41.8184478551838,73.2294362384225 41.2650217194684,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.96060044865294 55.9382939511593,5.6692453115051 56.0150275120673,6.38482117645567 56.0880973218335,7.10720375678705 56.157400475677,7.83625773770865 56.2228347173136,8.5718366560563 56.2842986378254,9.31378281572326 56.3416918804739,10.0619272412891 56.3949153508462,10.8160896721679 56.4438714316548,11.5760785994189 56.4884642014437,12.3416913471456 56.528599656387,13.1127142001617 56.5641859343031,13.8889225793161 56.5951335399513,14.6700812655504 56.6213555706215,15.4559446734179 56.6427679409819,16.2462571744128 56.6592896061102,17.0407534700619 56.6708427815999,17.8391590143095 56.6773531596105,18.6411904842936 56.6787501197174,19.4465562981665 56.6749669334121,20.2549571781681 56.6659409611101,21.0660867567155 56.6516138405428,21.8796322228404 56.6319316654367,22.6952750058883 56.6068451534252,23.5126914929996 56.5763098021872,24.3315537765309 56.5402860328691,25.1515304272452 56.4987393199198,25.9722872888145 56.4516403065472,26.7934882889404 56.3989649050969,27.6147962622065 56.3406943817482,28.4358737796488 56.2768154250305,29.2563839799456 56.207320197769,30.0759913971174 56.1322063721813,30.8943627796619 56.0514771479657,31.7111678961496 55.9651412533344,32.5260803224592 55.8732129290618,33.3387782060384 55.7757118957345,34.1489450028345 55.6726633044969,34.9562701828379 55.5640976716962,35.7604499005266 55.4500507979282,36.5611876268714 55.3305636720814,37.3581947399687 55.2056823610617,38.1511910717861 55.0754578859583,38.9399054089486 54.9399460854787,39.7240759459355 54.7992074675415,40.5034506895044 54.6533070499613,41.277787813601 54.5023141912026,42.0468559644411 54.3463024122039,42.8104345158644 54.1853492102971,43.5683137754524 54.0195358662508,44.3202951422663 53.8489472454711,45.066191217402 53.6736715943821,45.8058258688602 53.4938003329924,46.5390342525062 53.3094278446299,47.2656627911282 53.1206512637978,47.985569113808 52.9275702630661,48.6986219579803 52.7302868398744,49.4047010366934 52.5289051040743,50.1036968736777 52.3235310669909,50.7955106088955 52.1142724327332,51.4800537772815 51.9012383924278,52.1572480633875 51.6845394219956,52.8270250346284 51.4642870840378,53.4893258557795 51.2405938343408,54.1441009873167 51.0135728334539,54.791309870105 50.7833377637434,55.4309205988438 50.5500026522693,56.0629095865715 50.3136816997866,56.6872612224038 50.0744891161178,57.3039675245588 49.8325389621028,57.9130277905821 49.5879449982851,58.5144482465496 49.3408205404539,59.1082416968843 49.091278322122,59.6944271762829 48.8394303639847,60.2730296051101 48.5853878503721,60.8440794494795 48.3292610126751,61.40761238711 48.0711590197009,61.9636689799149 47.8111898748862,62.5122943541616 47.5494603202768,63.0535378889195 47.2860757471584,63.5874529134047 47.0211401132109,64.1140964137264 46.7547558660387,64.6335287494427 46.4870238729191,65.1458133802426 46.2180433565991,65.6510166029904 45.9479118369597,66.1492072992903 45.6767250783612,66.6404566936622 45.4045770424768,67.1248381223566 45.1315598464143,67.6024268127789 44.8577637259253,68.0732996734467 44.5832770034983,68.5375350943572 44.3081860611283,68.9952127576034 44.0325753175599,69.4464134580461 43.7565272097974,69.8912189338183 43.4801221786779,70.3297117064144 43.2034386583077,70.7619749300985 42.9265530691615,71.1880922503468 42.6495398146493,71.6081476710291 42.3724712809597,72.0222254300221 42.0954178399905,72.4304098829428 41.8184478551841,72.8327853946822 41.5416276900883,73.2294362384225 41.2650217194687,73.6204465018146 40.9886923428039,74.0059 40.7127)

test/CDB_HelperTest.sql

@@ -0,0 +1,128 @@
+-- Test unique identifier creation with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', NULL);
+
+-- Test unique identifier creation with prefix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'relname', NULL);
+
+-- Test unique identifier creation with suffix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', '_suffix');
+
+-- Test unique identifier creation with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test unique identifier creation with prefix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargolar;
+
+-- Test unique identifier creation with suffix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+DROP TABLE largolargolargolargolargolargolargolargolargolargolar_suffix;
+
+-- Test unique identifier creation with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', NULL);
+
+-- Test unique identifier creation with prefix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piraña', NULL);
+
+-- Test unique identifier creation with suffix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', '_suffix');
+
+-- Test unique identifier creation with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test unique identifier creation with prefix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+DROP TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi;
+
+-- Test unique identifier creation with suffix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargolar text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length normal colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (largolargolargolargolargolargolargolargolargolargolar_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length UTF8 colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test _CDB_Octet_Truncate simple case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 5);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 6);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 7);

test/CDB_HelperTest_expect

@@ -0,0 +1,59 @@
+relname
+prefix_relname
+relname_suffix
+largolargolargolargolargolargolargolargolargolargolargolargo
+prefix_largolargolargolargolargolargolargolargolargolargolar
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargolar_0
+DROP TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix_0
+DROP TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+DROP TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+DROP TABLE
+CREATE TABLE
+colname
+prefix_colname
+colname_suffix
+largolargolargolargolargolargolargolargolargolargolargolargo
+prefix_largolargolargolargolargolargolargolargolargolargolar
+DROP TABLE
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargolar_0
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix_0
+DROP TABLE
+CREATE TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+DROP TABLE
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+DROP TABLE
+pira
+pirañ
+piraña

test/CDB_QuotaTest.sql

@@ -1,22 +1,31 @@
 set client_min_messages to error;
 \set VERBOSITY default
 
+-- See the dice
+SELECT setseed(0.5);
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
       EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(2049,4096);
+INSERT INTO big SELECT generate_series(4097,6144);
+INSERT INTO big SELECT generate_series(6145,8192);
+-- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
+-- END Test for #108
+SELECT setseed(0.9);
 SELECT CDB_SetUserQuotaInBytes(2);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8194);
 DROP TABLE big;
 set client_min_messages to NOTICE;
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,15 +1,20 @@
 SET
+
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
+big
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+454656
+909312
+0
 
-INSERT 0 2048
-INSERT 0 2048
-INSERT 0 2048
 2
-ERROR:  Quota exceeded by 567.998046875KB
+ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/CDB_StatsTest.sql

@@ -0,0 +1,13 @@
+-- continuous uniform distribution has kurtosis = -6/5, skewness = 0.0
+-- http://mathworld.wolfram.com/UniformDistribution.html
+set client_min_messages to ERROR;
+
+WITH dist AS (
+  SELECT generate_series(0,10000)::numeric / 10000.0 i
+)
+SELECT
+  abs(CDB_Kurtosis(array_agg(i)) + 1.2) < 1e-3 AS kurtosis,
+  abs(CDB_Skewness(array_agg(i))) < 1e-3 AS skewness
+FROM dist;
+
+set client_min_messages to NOTICE;

test/CDB_StatsTest_expect

@@ -0,0 +1,3 @@
+SET
+t|t
+SET

test/extension/run_at_cartodb_schema.sql

@@ -1,4 +1,6 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
 \i scripts-available/CDB_TableMetadata.sql
+\i scripts-available/CDB_ColumnNames.sql
+\i scripts-available/CDB_ColumnType.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -123,7 +123,7 @@ function create_role_and_schema() {
 
 function drop_role_and_schema() {
     local ROLE=$1
-    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "DROP SCHEMA \"${ROLE}\" CASCADE;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
     sql "DROP ROLE \"${ROLE}\";"
 }
@@ -178,6 +178,7 @@ function setup() {
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
     sql "CREATE EXTENSION postgis;"
+    sql "CREATE EXTENSION plpythonu;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
@@ -209,8 +210,8 @@ function tear_down() {
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
-    sql 'DROP SCHEMA cdb_testmember_1;'
-    sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_testmember_1 CASCADE;'
+    sql 'DROP SCHEMA cdb_testmember_2 CASCADE;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
@@ -227,7 +228,12 @@ function run_tests() {
     local TESTS
     if [[ $# -ge 1 ]]
     then
-        TESTS="$@"
+        if [[ $# -eq 1 ]]
+        then
+            TESTS=`cat $0 | grep -o "$1[^\(]*"`
+        else
+            TESTS="$@"
+        fi
     else
         TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
     fi
@@ -337,6 +343,92 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_column_names() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
+
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "carto"
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "db"
+
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames'::regclass) c) as s" should "carto"
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_2.table_cnames') c) as s" should "db"
+
+    # Using schema from owner
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" should "carto"
+
+    ## it's not possible to get column names from a table where you don't have permissions
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" fails
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_cnames'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_cnames'
+}
+
+function test_cdb_column_type() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_ctype(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_ctype(c text, a text, r text, t text, o text);'
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "integer"
+    sql cdb_testmember_2 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "text"
+
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_1.table_ctype', 'c')" should "integer"
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_2.table_ctype', 'c')" should "text"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_ctype'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_ctype'
+}
+
+function test_cdb_querytables_schema_and_table_names_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE SCHEMA "foo.bar";'
+    sql postgres 'CREATE TABLE "foo.bar"."c.a.r.t.o.d.b" (a int);'
+    sql postgres 'INSERT INTO "foo.bar"."c.a.r.t.o.d.b" values (1);'
+    sql postgres 'SELECT a FROM "foo.bar"."c.a.r.t.o.d.b";' should 1
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+
+    sql postgres 'DROP TABLE "foo.bar"."c.a.r.t.o.d.b";'
+    sql postgres 'DROP SCHEMA "foo.bar";'
+}
+
+function test_cdb_querytables_table_name_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE "w.a.d.u.s" (a int);';
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+
+    sql postgres 'DROP TABLE "w.a.d.u.s";';
+}
+
+function test_cdb_querytables_happy_cases() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE wadus (a int);';
+    sql postgres 'CREATE TABLE "FOOBAR" (a int);';
+    sql postgres 'CREATE SCHEMA foo;'
+    sql postgres 'CREATE TABLE foo.wadus (a int);';
+
+    ## See how it does NOT quote anything here
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+
+    ## But it quotes when it's needed even if table name has no dots but was created with quotes
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "FOOBAR"$q$);' should '{"public.\"FOOBAR\""}'
+
+    sql postgres 'DROP TABLE wadus;'
+    sql postgres 'DROP TABLE "FOOBAR";'
+    sql postgres 'DROP TABLE foo.wadus;'
+    sql postgres 'DROP SCHEMA foo;'
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@

test/organization/test.sh

@@ -28,6 +28,7 @@ function clear_partial_result() {
 function sql() {
     local ROLE
     local QUERY
+    ERROR_OUTPUT_FILE='/tmp/test_error.log'
     if [[ $# -ge 2 ]]
     then
         ROLE="$1"
@@ -37,15 +38,40 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as ${ROLE}"
-      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}' as ${ROLE}"
+        RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     else
-      log_debug "Executing query '${QUERY}'"
-      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}'"
+        RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     fi
     CODERESULT=$?
+    ERROR_OUTPUT=`cat "${ERROR_OUTPUT_FILE}"`
+    rm ${ERROR_OUTPUT_FILE}
 
-    echo ${RESULT}
+    echo -n "> Code Result: "
+    echo -n ${CODERESULT}
+    echo -n "; Result: "
+    echo -n ${RESULT}
+    echo -n "; Error output: "
+    echo -n ${ERROR_OUTPUT}
+
+    # Some warnings should actually be failures
+    if [[ ${CODERESULT} == "0" ]]
+    then
+        case "${ERROR_OUTPUT}" in
+            WARNING:*no*privileges*were*granted*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES GRANTING WARNING"
+                CODERESULT=1
+            ;;
+            WARNING:*no*privileges*could*be*revoked*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES REVOKING WARNING"
+                CODERESULT=1
+            ;;
+            *) ;;
+        esac
+        echo -n "; Code result after warnings: "
+        echo -n ${CODERESULT}
+    fi
     echo
 
     if [[ ${CODERESULT} -ne 0 ]]
@@ -133,13 +159,18 @@ function create_table() {
 function setup() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
+    sql "CREATE EXTENSION plpythonu;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
-
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups_API.sql
 
     log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_org_admin
+    sql "SELECT cartodb.CDB_Organization_AddAdmin('cdb_org_admin');"
     create_role_and_schema cdb_testmember_1
     create_role_and_schema cdb_testmember_2
     sql "CREATE ROLE publicuser LOGIN;"
@@ -152,8 +183,16 @@ function setup() {
     create_table cdb_testmember_2 bar
     sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3), (4), (5);'
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_a_tmp')"
+    sql "SELECT cartodb.CDB_Group_RenameGroup('group_a_tmp', 'group_a')"
+
+    sql "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_b')"
 }
 
+
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -162,19 +201,29 @@ function tear_down() {
     sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
     sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
 
+    sql "select cartodb.CDB_Group_DropGroup('group_b')"
+
+    sql "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "select cartodb.CDB_Group_DropGroup('group_a')"
+    sql "SELECT cartodb.CDB_Organization_RemoveAdmin('cdb_org_admin');"
+
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
     sql 'DROP SCHEMA cdb_testmember_1;'
     sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_org_admin;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_org_admin;"
 
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
     sql 'DROP ROLE publicuser;'
+    sql 'DROP ROLE cdb_org_admin;'
 
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
@@ -198,6 +247,7 @@ function run_tests() {
         echo "####################################################################"
         clear_partial_result
         setup
+        log_info "############################# TESTS #############################"
         eval ${t}
         if [[ ${PARTIALOK} -ne 0 ]]
         then
@@ -330,44 +380,167 @@ function test_user_can_read_when_it_has_permission_after_organization_permission
 }
 
 function test_cdb_querytables_returns_schema_and_table_name() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
 }
 
 function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_usertables_should_work_with_orgusers() {
-    sql "GRANT USAGE ON SCHEMA cartodb TO publicuser;"
-    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
+    # This test validates the changes proposed in https://github.com/CartoDB/cartodb/pull/5021
+
+    # create tables
     sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
-    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+    sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
     sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
+    
+    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+
+
+    # this is what we need to make public tables available in CDB_UserTables
+    sql postgres "grant publicuser to cdb_testmember_1;"
+    sql postgres "grant publicuser to cdb_testmember_2;"
+
+
+    # this is required to enable select from other schema
+    sql postgres "GRANT USAGE ON SCHEMA cdb_testmember_1 TO publicuser";
+
+
+    # test CDB_UserTables with publicuser
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
     sql publicuser "SELECT count(*) FROM CDB_UserTables('all')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('public')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    sql publicuser "SELECT * FROM CDB_UserTables('all')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('private')" should ""
     # the following tests are for https://github.com/CartoDB/cartodb-postgresql/issues/98
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('all')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('public')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    # cdb_testmember_2 is already owner of `bar` table
+    sql cdb_testmember_2 "select string_agg(t,',') from (select cdb_usertables('all') t order by t) as s" should "bar,test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('private')" should "bar"
+
+    # test cdb_testmember_2 can select from cdb_testmember_1's public table
+    sql cdb_testmember_2 "SELECT * FROM cdb_testmember_1.test_perms_pub" should 1
 
     sql cdb_testmember_1 "DROP TABLE test_perms_pub"
     sql cdb_testmember_1 "DROP TABLE test_perms_priv"
 }
 
+function test_CDB_Group_Table_GrantRead_should_grant_select_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_CDB_Group_Table_GrantReadWrite_should_grant_insert_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_cant_be_used_by_normal_members() {
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_CreateGroup('group_x_1');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RenameGroup('group_a', 'group_x_2');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_DropGroup('group_a');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_2']);" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1']);" fails
+}
+
+function test_group_permission_functions_cant_be_used_by_normal_members() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_can_be_used_by_org_admin() {
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_CreateGroup('group_x_tmp');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RenameGroup('group_x_tmp', 'group_x');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_AddUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RemoveUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    # TODO: workaround superadmin limitation
+    sql "SELECT cartodb.CDB_Group_DropGroup('group_x');"
+}
+
+function test_org_admin_cant_grant_permissions_on_tables_he_does_not_own() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_valid_group_names() {
+    sql postgres "select cartodb._CDB_Group_GroupRole('group_1$_a');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('GROUP_1$_A');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('_group_1$_a');"
+}
+
+function test_administrator_name_generation() {
+    sql postgres "select cartodb._CDB_Organization_Admin_Role_Name();"
+}
+
+function test_conf() {
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('test_conf', '{ \"a_key\": \"test_val\" }')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should '{ "a_key": "test_val" }'
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_RemoveConf('test_conf')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+}
 
 #################################################### TESTS END HERE ####################################################