Release 0.13.0

Overviews (new feature)

.gitignore

@@ -5,4 +5,5 @@ results/
 regression.*
 expected/test
 sql/test
-.idea/*
+.idea/*
+*.swp

.travis.yml

@@ -9,14 +9,6 @@ before_install:
   - sudo apt-get update
   - sudo apt-get install -q postgresql-server-dev-9.3
   - sudo apt-get install -q postgresql-plpython-9.3
-  # Install schema_triggers
-  - hg clone https://bitbucket.org/malloclabs/pg_schema_triggers &&
-    cd pg_schema_triggers && make && sudo make install && cd -
-  # Preload schema_triggers module
-  # NOTE: might change if we make it part of the installcheck instead
-  - echo "shared_preload_libraries = 'schema_triggers.so'" |
-    sudo tee -a /etc/postgresql/9.3/main/postgresql.conf &&
-    sudo service postgresql restart
 
 script:
   - make

CONTRIBUTING.md

@@ -44,17 +44,21 @@ the extension into your test database.
 
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires cheating with PostgreSQL
-so to enforce re-load of the scripts. To help with cheating, "make install"
+as to enforce the scripts to reload. To help with cheating, "make install"
 also installs migration scripts to go from "V" to "V"next and from "V"next
 to "V". Example to upgrade a 0.2.0dev version:
 
 ```sql
-ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
+ALTER EXTENSION cartodb UPDATE TO '0.2.0next';
 ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
 ```
-
 Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
 
 ```sql
 SELECT cartodb.cdb_extension_reload();
 ```
+
+A useful query:
+```sql
+SELECT * FROM pg_extension_update_paths('cartodb') WHERE path IS NOT NULL AND source = cdb_version();
+```

Makefile

@@ -1,16 +1,16 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.8.2
+EXTVERSION = 0.13.0
 
 SED = sed
 
 CDBSCRIPTS = \
   scripts-enabled/*.sql \
   scripts-available/CDB_SearchPath.sql \
-  scripts-available/CDB_DDLTriggers.sql \
   scripts-available/CDB_ExtensionPost.sql \
   scripts-available/CDB_ExtensionUtils.sql \
+  scripts-available/CDB_Helper.sql \
   $(END)
 
 UPGRADABLE = \
@@ -41,6 +41,23 @@ UPGRADABLE = \
   0.7.4 \
   0.8.0 \
   0.8.1 \
+  0.8.2 \
+  0.9.0 \
+  0.9.1 \
+  0.9.2 \
+  0.9.3 \
+  0.9.4 \
+  0.10.0 \
+  0.10.1 \
+  0.10.2 \
+  0.11.0 \
+  0.11.1 \
+  0.11.2 \
+  0.11.3 \
+  0.11.4 \
+	0.11.5 \
+	0.12.0 \
+	0.13.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -62,10 +79,9 @@ DATA_built = \
 EXTRA_CLEAN = cartodb_version.sql
 
 DOCS = README.md
-REGRESS_NEW = test_ddl_triggers
 REGRESS_OLD = $(wildcard test/*.sql)
 REGRESS_LEGACY = $(REGRESS_OLD:.sql=)
-REGRESS = test_setup $(REGRESS_NEW) $(REGRESS_LEGACY)
+REGRESS = test_setup $(REGRESS_LEGACY)
 
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)

NEWS.md

@@ -1,3 +1,98 @@
+0.13.0 (2016-01-29)
+* Add CDB_CreateOverviews, CDB_DropOverviews and CDB_Overviews for vector overviews support. [185](https://github.com/CartoDB/cartodb-postgresql/pull/185)
+* Convert some simple functions from plpgsql to sql. [188](https://github.com/CartoDB/cartodb-postgresql/pull/188)
+
+0.12.0 (2016-01-27)
+-------------------
+* Remove schema_triggers extension dependency, to ensure compatibility with PostgreSQL 9.5. [#190](https://github.com/CartoDB/cartodb-postgresql/pull/190)
+* Remove DDL trigger functions (unused by CartoDB).
+
+0.11.5 (2015-11-27)
+-------------------
+* Disable log invalidation time [#178](https://github.com/CartoDB/cartodb-postgresql/pull/178)
+
+0.11.4 (2015-11-24)
+-------------------
+* Fix for existing PK cartodb_id problem [#174](https://github.com/CartoDB/cartodb-postgresql/issues/174)
+* Add cartodbfication support for column names with embedded points to fix [#6114](https://github.com/CartoDB/cartodb/issues/6114)
+* Add CDB_GreatCircle for creating great circle routes between two points [#171](https://github.com/CartoDB/cartodb-postgresql/pull/171)
+* Fix to prevent cartodbfication problems [#155](https://github.com/CartoDB/cartodb-postgresql/issues/155)
+
+0.11.3 (2015-10-27)
+-------------------
+* Added CDB_Helper.sql [#173](https://github.com/CartoDB/cartodb-postgresql/pull/173)
+* Added `_CDB_Unique_Identifier` for creating UTF8 aware unique identifiers
+* Added `_CDB_Unique_Column_Identifier` for creating UTF8 aware unique identifiers for columns
+* Added `_CDB_Octet_Truncate` that truncates text to a certain amount of octets.
+
+0.11.2 (2015-10-19)
+-------------------
+* Fix schema not being specified on pg_get_serial_sequence [#170](https://github.com/CartoDB/cartodb-postgresql/pull/170)
+* Log invalidation function call duration in seconds [#163](https://github.com/CartoDB/cartodb-postgresql/pull/163)
+
+0.11.1 (2015-10-06)
+-------------------
+* Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169)
+* cartodbfy now discards cartodb_id candidates that contain nulls [#148](https://github.com/CartoDB/cartodb-postgresql/issues/148)
+
+0.11.0 (2015-09-dd)
+-------------------
+* Groups API
+
+0.10.2 (2015-09-24)
+-------------------
+* Add back the `DROP FUNCTION IF EXISTS CDB_UserTables(text);` to be able to upgrade from `0.7.3` upward [#160](https://github.com/CartoDB/cartodb-postgresql/issues/160)
+
+0.10.1 (2015-09-16)
+-------------------
+* Get back the `update_updated_at` function (still used by old tables) [#143](https://github.com/CartoDB/cartodb-postgresql/pull/143)
+* Fix for CDB_StatsTest.sql test failing randomly [#144](https://github.com/CartoDB/cartodb-postgresql/issues/144)
+* Fix for table cartodbfy'ed without default seq value [#138](https://github.com/CartoDB/cartodb-postgresql/issues/138)
+* Fix for cartodbfy error column `the_geom` already exists [#141](https://github.com/CartoDB/cartodb-postgresql/issues/141)
+* Fix for columns with geometry cartodbfy'ed without SRID [#154](https://github.com/CartoDB/cartodb-postgresql/issues/154)
+
+0.10.0 (2015-09-07)
+-----------------
+* Quote schema and table names returned by CDB_QueryTables [#134](https://github.com/CartoDB/cartodb-postgresql/pull/134). Use quote_ident to quote schema and table names when necessary.
+* Fixed CDB_ColumnNames [#122](https://github.com/CartoDB/cartodb-postgresql/issues/122) and CDB_ColumnType [#130](https://github.com/CartoDB/cartodb-postgresql/issues/130) should honor regclass, returning columns for just the table in the schema and not in any other one [#131](https://github.com/CartoDB/cartodb-postgresql/pull/131).
+* Add kurtosis and skewness [#124](https://github.com/CartoDB/cartodb-postgresql/pull/124).
+* Removed `DROP FUNCTION IF EXISTS cdb_usertables(text);` [#129](https://github.com/CartoDB/cartodb-postgresql/pull/129). This was needed for upgrading between 0.7.4 to 0.8.0 but is no longer needed.
+
+0.9.4 (2015-08-28)
+------------------
+* Fixed issue with indices when renaming tables [#123](https://github.com/CartoDB/cartodb-postgresql/issues/123)
+
+0.9.3 (2015-08-27)
+------------------
+* Modify sampling of quota trigger [#126](https://github.com/CartoDB/cartodb-postgresql/issues/126)
+
+0.9.2 (2015-08-24)
+------------------
+* Fix for `the_geom` column present but not SRID (EWKT) and other corner cases [#121](https://github.com/CartoDB/cartodb-postgresql/pull/121)
+
+0.9.1 (2015-08-19)
+------------------
+* Fix for transformation to webmercator in corner cases [#116](https://github.com/CartoDB/cartodb-postgresql/issues/116)
+
+0.9.0 (2015-08-19)
+------------------
+* Re-implementation of `CDB_CartodbfyTable` functions
+  - The signature of the main function changes to
+    ```
+    FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
+    RETURNS REGCLASS
+    ```
+    - The `destschema` does not need to match the origin schema of `reloid`
+	- It returns the `regclass` of the cartodbfy'ed table, if it needs to be rewritten.
+  - There are many optimizations
+  - The columns `created_at` and `updated_at` will no longer be added
+* Fix for CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` #110
+* Review test to validate permissions in public tables [#112](https://github.com/CartoDB/cartodb-postgresql/pull/112)
+
+0.8.3 (2015-08-14)
+------------------
+* Fixes CDB_UserDataSize failing due `ERROR: relation "*" does not exist.` [#108](https://github.com/CartoDB/cartodb-postgresql/issues/108)
+
 0.8.2 (2015-07-27)
 ------------------
 * Fix for CDB_UserTables returning wrong listings when publicuser is used

README.md

@@ -6,16 +6,13 @@ cartodb-postgresql
 
 PostgreSQL extension for CartoDB
 
-See https://github.com/CartoDB/cartodb/wiki/CartoDB-PostgreSQL-extension
+See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/wiki).
 
 Dependencies
 ------------
 
  * PostgreSQL 9.3+ (with plpythonu extension and xml support)
  * [PostGIS extension](http://postgis.net) 
- * [Schema triggers extension]
-   (https://bitbucket.org/malloclabs/pg_schema_triggers)
-   (or [fork](https://github.com/CartoDB/pg_schema_triggers))
 
 Install
 -------
@@ -31,12 +28,10 @@ Test installation
 make installcheck
 ```
 
-NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
-      installation of schema_triggers: you need to add ``schema_triggers.so``
-      to the ``shared_preload_libraries`` setting in postgresql.conf !
-
 NOTE: you need to run the installcheck as a superuser, use PGUSER
       env variable if needed, like: PGUSER=postgres make installcheck
+      
+NOTE: the tests need to run against a **clean postgres instance**, if you have some roles already created test will likely fail due `publicuser` not being dropped.
 
 Enable database
 ---------------
@@ -45,7 +40,6 @@ In a database that needs to be turned into a "cartodb" user database, run:
 
 ```sql
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION cartodb;
 ```
 
@@ -61,7 +55,6 @@ be in the "cartodb" schema.
 
 ```sql
 CREATE EXTENSION postgis FROM unpackaged;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION cartodb FROM unpackaged;
 ```
 
@@ -96,3 +89,8 @@ ls `pg_config --sharedir`/extension/cartodb*
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires special steps documented
 in the CONTRIBUTING document, under "Testing changes live".
+
+Limitations
+-----------
+
+- The main schema of an organization user must have one only owner (the user).

cartodb.control.in

@@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
 superuser = true
 relocatable = false
 schema = cartodb
-requires = 'plpythonu, schema_triggers, postgis'
+requires = 'plpythonu, postgis'

doc/CDB_GreatCircle.md

@@ -0,0 +1,16 @@
+Based on Paul Ramsey's [blog post](http://blog.cartodb.com/jets-and-datelines/).
+#### Using the function
+
+Creates a great circle line.
+
+```sql
+SELECT CDB_GreatCircle(start_point, end_point) FROM table_name
+-- Results a line reprsenting the great circle between the two points
+```
+
+#### Arguments
+
+CDB_GreatCircle(start_point, end_point)
+
+* **start_point** ST_Point indicating the start of the line.
+* **end_point** ST_point indicating the end of the line.

doc/CDB_Overviews.md

@@ -0,0 +1,105 @@
+Overviews are tables that represent a *reduced* version of a dataset intended
+for efficient rendering at certain zoom levels while preserving the
+general visual appearance of the complete dataset.
+
+The *reduction* consists in a fewer number of records
+(while each overview record may represent an aggregation of multiple records)
+and/or simplified record geometries.
+
+Overviews are created through the `CDB_CreateOverviews`.
+The statement timeout may need to be adjusted before using this function,
+as overview creation for large tables is a time-consuming operation.
+
+The `CDB_Overviews` function can be used determine what overview tables
+exist for a given dataset table and which zoom levels correspond to it.
+
+The `CDB_DropOverviews` remove a dataset's existing overviews.
+
+### CDB_CreateOverviews
+
+Create overviews for vector dataset.
+
+#### Using the function
+
+The table for which overviews will be generated should be
+a Cartodbfied dataset with vector geometry.
+
+```sql
+SELECT CDB_CreateOverviews('table_name');
+--- Generates overview tables for the dataset
+```
+
+#### Arguments
+
+CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
+
+* **table_name** regclass, table for which overviews will be generated
+* **ref_z_strategy** regproc, optional function that provides
+  the Z-scale strategy.
+  It returns the base Z level for the dataset.
+  It should have these arguments:
+  - **table_name** regclass, table to compute the reference Z scale for
+* **reduction_strategy** regproc, optional function that provides
+  the reduction strategy to generate an overview table from a table
+  for a smaller scale (higher Z number).
+  It returns the name of the generated table.
+  It should have these arguments:
+  - **base_table_name** regclass, base table to be reduced.
+  - **base_z** integer, base Z level assigned to the base table.
+  - **overview_z** integer, Z level for which to generate the overview.
+
+### CDB_Overviews
+
+Obtain overview metadata for a given table (existing overviews).
+The returned relation will be empty if the table has no overviews.
+
+The function can be applied to a single table:
+
+```sql
+SELECT CDB_Overviews('table_name');
+--- Return existing overview Z levels and corresponding tables
+```
+
+Or to multiple tables passed as an array; this can be used
+to obtain the overviews that can be applied to a query by
+combining it with `CDB_QueryTablesText`:
+
+```sql
+SELECT CDB_Overviews(CDB_QueryTablesText('SELECT * FROM table1, table2'));
+--- Return existing overview Z levels and corresponding tables
+```
+
+The result of `CDB_Overviews` has three columns:
+
+| base_table | z | overview_table |
+|------------+---+----------------|
+| table1     | 1 | table1_ov1     |
+| table1     | 2 | table1_ov2     |
+| table1     | 4 | table1_ov4     |
+| table2     | 1 | table1_ov1     |
+| table2     | 2 | table1_ov2     |
+
+#### Arguments
+
+CDB_Overviews(table_name)
+
+* **table_name** regclass, oid of table to obtain existing overviews for
+
+CDB_Overviews(table_names)
+
+* **table_names** regclass[], array of table oids
+
+
+### CDB_DropOverviews
+
+Remove the overviews of a table, if present.
+
+```sql
+SELECT CDB_DropOverviews('table_name');
+```
+
+#### Arguments
+
+CDB_Overviews(table_name)
+
+* **table_name** regclass, table for which to drop existing overviews.

doc/README.md

@@ -20,29 +20,4 @@ The CartoDB PostgreSQL extension is a module to load into each CartoDB user data
 
 # Checks
 
-No user other than the superuser should be allowed to change `statement_timeout` for the session (SET statement_timeout disallowed).
-
 User tables need to match certain structure criteria (See [[CartoDB-user-table]]) so the extension should provide a mean to enforce such structure everytime an attempt to change structure is encountered. 
-
-# Events
-
-The events we want some function to be called upon are:
-
-| event                  | arguments to handler function        | function duty                    | OK* |
-|------------------------|--------------------------------------|----------------------------------|-----|
-| SET variable           | name of variable                     | forbid changing some vars        |     |
-| RENAME table           | old and new name + oid of the table  | flush cache                      |     |
-| ADD/DROP/ALTER column  | oid of the table                     | flush cache, cartodbfy, upd meta |  Y  |
-| DISABLE/DROP trigger   | oid of table, name of trigger        | cartodbfy or forbid              |     |
-| DROP table             | oid of the table                     | flush cache and metadata         |  Y  |
-| CREATE table           | oid of the table                     | cartodby, upd metadata           |  Y  |
-| GRANT                  | oid of table, privilege, role        | flush cache, upd metadata        |
-| REVOKE                 | oid of table, privilege, role        | flush cache, upd metadata        |
-
-
-* event available by installing https://github.com/CartoDB/pg_schema_triggers
-
-At this stage we don't need more than this, but the number of events and the number of arguments passed to the handler function may expand in the future, so the extension should be written in a way to easily allow that.
-
-Some of the handler will need to act _after_ the statement is completed (CREATE TABLE, for example).
-

doc/cartodbfy-requirements.rst

@@ -0,0 +1,59 @@
+Introduction
+============
+
+This document aims at describing what cartodbfy is and what its formal requirements are, with the following goals in mind:
+
+- clarify what are the expectations of the "cartodbfycation process".
+- define an important part of what should be a stable, public API
+- allow for better testing, which should in turn...
+- ...ease modifications and increase quality of the code
+
+
+
+What is the cartodbfycation
+===========================
+
+The cartodbfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
+
+
+
+Valid CartoDB tables
+====================
+
+A valid CartoDB table shall meet the following conditions:
+
+- Have a ``cartodb_id`` integer column as primary key with a sequence as default value
+- Have a ``the_geom`` column of type ``Geometry`` with SRID 4326
+- Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857
+- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync
+
+Additionally, a CartoDB table can contain other columns.
+
+
+
+High level requirements
+=======================
+
+Here is a list of high level requirments for the public function ``CDB_CartodbfyTable()``:
+
+- A call to ``CDB_CartodbfyTable()`` shall modify/rewrite the table and produce a valid CartoDB table with the same name.
+- A call to ``CDB_CartodbfyTable()`` shall cause the registration of the table into the platform
+- It shall be idempotent, meaning that successive calls to ``CDB_CartodbfyTable()`` shall not produce any visible effect in the system.
+- If there's a column containing a geometry, it shall be used to generate ``the_geom`` and the ``the_geom_webmercator`` columns.
+- Exporting and re-importing the same table in CartoDB shall produce equivalent tables, with the same features associated to the same ``cartodb_id``'s.
+
+Note that there should be only one feature per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
+
+
+
+Low-level requirements
+======================
+
+- If the original table contains a valid (unique and not null) ``cartodb_id`` column, it shall be used
+- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
+- A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata``
+- A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id``
+- A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator``
+- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns
+
+

expected/test_ddl_triggers.out

@@ -1,276 +0,0 @@
-\set VERBOSITY terse
--- Set user quota to infinite
-SELECT CDB_SetUserQuotaInBytes(0);
- cdb_setuserquotainbytes 
--------------------------
-                       0
-(1 row)
-
--- Enable ddl triggers
-SELECT cartodb.cdb_enable_ddl_hooks();
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
- cdb_enable_ddl_hooks 
-----------------------
- 
-(1 row)
-
-create schema c;
-SELECT CDB_SetUserQuotaInBytes('c', 0);
- cdb_setuserquotainbytes 
--------------------------
-                       0
-(1 row)
-
-CREATE USER cartodb_postgresql_unpriv_user;
-GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
---SELECT session_user, current_user;
-----------------------
--- CREATE TABLE
-----------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select 1 as i INTO c.t3;
-NOTICE:  trigger "track_updates" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "test_quota" for table "c.t3" does not exist, skipping
-NOTICE:  trigger "test_quota_per_row" for table "c.t3" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
-NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i 
-------------+-----+------+----------+----------------------+---
-          1 | t   | t    |          |                      | 1
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)) as age
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
- tabname | age 
----------+-----
- c.t3    |   0
-(1 row)
-
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
--- Table with cartodb_id field, see
--- http://github.com/CartoDB/cartodb-postgresql/issues/32
-select 1 as cartodb_id INTO c.t4;
-NOTICE:  trigger "track_updates" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "update_the_geom_webmercator_trigger" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "update_updated_at_trigger" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "test_quota" for table "c.t4" does not exist, skipping
-NOTICE:  trigger "test_quota_per_row" for table "c.t4" does not exist, skipping
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
-NOTICE:  Trying to recover data from _cartodb_id0 column
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
-NOTICE:  cdb_invalidate_varnish(c.t4) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator 
-------------+-----+------+----------+----------------------
-          1 | t   | t    |          | 
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now() - updated_at)) as age
-FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
- tabname | age 
----------+-----
- c.t4    |   0
-(1 row)
-
-----------------------------
--- ALTER TABLE RENAME COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
- pg_sleep 
-----------
- 
-(1 row)
-
-alter table c.t3 rename column the_geom_webmercator to webmerc;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
-NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc 
-------------+-----+------+----------+----------------------+---+---------
-          1 | t   | t    |          |                      | 1 | 
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
- tabname | agecs 
----------+-------
- c.t3    |     0
-(1 row)
-
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
- pg_sleep 
-----------
- 
-(1 row)
-
-alter table c.t3 rename column the_geom_webmercator to webmerc2;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
-NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
- tabname | agecs 
----------+-------
- c.t3    |     0
-(1 row)
-
-----------------------------
--- ALTER TABLE DROP COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
- pg_sleep 
-----------
- 
-(1 row)
-
-alter table c.t3 drop column the_geom_webmercator;
-NOTICE:  Column cartodb_id already exists
-NOTICE:  Column created_at already exists
-NOTICE:  Column updated_at already exists
-NOTICE:  Column the_geom already exists
-NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
-NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
-NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_drop_column" does not exist, skipping
-NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
-NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 
-------------+-----+------+----------+----------------------+---+---------+----------
-          1 | t   | t    |          |                      | 1 |         | 
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
- tabname | agecs 
----------+-------
- c.t3    |     0
-(1 row)
-
-----------------------------
--- ALTER TABLE ADD COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
- pg_sleep 
-----------
- 
-(1 row)
-
-alter table c.t3 add column id2 int;
-NOTICE:  cdb_invalidate_varnish(c.t3) called
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
- cartodb_id | c=u | u<1s | the_geom | the_geom_webmercator | i | webmerc | webmerc2 | id2 
-------------+-----+------+----------+----------------------+---+---------+----------+-----
-          1 | t   | t    |          |                      | 1 |         |          |    
-(1 row)
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
- tabname | agecs 
----------+-------
- c.t3    |     0
-(1 row)
-
-----------------------------
--- DROP TABLE
-----------------------------
-RESET SESSION AUTHORIZATION;
-drop schema c cascade;
-NOTICE:  drop cascades to 3 other objects
-select count(*) from CDB_TableMetadata;
- count 
--------
-     0
-(1 row)
-
-DROP USER cartodb_postgresql_unpriv_user;
-DROP FUNCTION _CDB_UserQuotaInBytes();

expected/test_setup.out

@@ -1,5 +1,4 @@
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)

scripts-available/CDB_ColumnNames.sql

@@ -3,12 +3,14 @@ CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
 
-    SELECT column_name 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || '';
-         
+    SELECT c.column_name
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace
+      ORDER BY ordinal_position;
+
 $$ LANGUAGE SQL;
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_ColumnType.sql

@@ -3,12 +3,13 @@ CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
 
-    SELECT data_type 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || ''
-        AND column_name = '' || quote_ident($2) || '';
+    SELECT c.data_type
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND column_name = $2
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_Conf.sql

@@ -0,0 +1,48 @@
+----------------------------------
+-- CONF MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by superadmin user.
+-- Functions needing reading configuration should use SECURITY DEFINER.
+----------------------------------
+
+-- This will trigger NOTICE if cartodb.CDB_CONF already exists
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    CREATE TABLE IF NOT EXISTS cartodb.CDB_CONF ( KEY TEXT PRIMARY KEY, VALUE JSON NOT NULL );
+END
+$$;
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.CDB_CONF', '');
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cartodb.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_GetConf(key text)
+    RETURNS JSON AS $$
+DECLARE
+    value JSON;
+BEGIN
+    EXECUTE 'SELECT VALUE FROM cartodb.CDB_CONF WHERE KEY = $1;' INTO value USING key;
+    RETURN value;
+END
+$$ LANGUAGE PLPGSQL STABLE;

scripts-available/CDB_DDLTriggers.sql

@@ -1,196 +0,0 @@
--- Table creation
--- {
-CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
-RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
-DECLARE
-  event_info RECORD;
-  rel_namespace TEXT;
-BEGIN
-  event_info := schema_triggers.get_relation_create_eventinfo();
-
-  -- We're only interested in real relations
-  IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
-
-  SELECT nspname FROM pg_namespace WHERE oid=(event_info.new).relnamespace INTO rel_namespace;
-
-  RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relname::TEXT, rel_namespace, (event_info.new).relnamespace;
-
-  -- We don't want to react to alters triggered by superuser,
-  IF current_setting('is_superuser') = 'on' THEN
-    RAISE DEBUG 'no ddl trigger for superuser';
-    RETURN;
-  END IF;
-
-  PERFORM cartodb.cdb_disable_ddl_hooks();
-
-  -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
-
-  PERFORM cartodb.cdb_enable_ddl_hooks();
-
-  RAISE DEBUG 'Inserting into cartodb.CDB_TableMetadata';
-
-  -- Add entry to CDB_TableMetadata (should CartodbfyTable do this?)
-  INSERT INTO cartodb.CDB_TableMetadata(tabname,updated_at) 
-  VALUES (event_info.relation, now());
-
-END; $$;
--- }
-
--- Table drop
--- {
-CREATE OR REPLACE FUNCTION cartodb.cdb_handle_drop_table ()
-RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
-DECLARE
-  event_info RECORD;
-BEGIN
-  event_info := schema_triggers.get_relation_drop_eventinfo();
-
-  -- We're only interested in real relations
-  IF (event_info.old).relkind != 'r' THEN RETURN; END IF;
-
-  RAISE DEBUG 'Relation % of kind % dropped from namespace oid %',
-	 event_info.old_relation_oid, (event_info.old).relkind, (event_info.old).relnamespace;
-
-  -- delete record from CDB_TableMetadata (should invalidate varnish)
-  DELETE FROM cartodb.CDB_TableMetadata WHERE tabname = event_info.old_relation_oid;
-
-END; $$;
--- }
-
--- Column alter
--- {
-CREATE OR REPLACE FUNCTION cartodb.cdb_handle_alter_column ()
-RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
-DECLARE
-  event_info RECORD;
-  rel RECORD;
-  rel_namespace TEXT;
-BEGIN
-  event_info := schema_triggers.get_column_alter_eventinfo();
-
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
-
-  RAISE DEBUG 'Column % altered by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
-
-  -- We're only interested in real relations
-  IF rel.relkind != 'r' THEN RETURN; END IF;
-
-  -- We don't want to react to alters triggered by superuser,
-  IF current_setting('is_superuser') = 'on' THEN
-    RAISE DEBUG 'no ddl trigger for superuser';
-    RETURN;
-  END IF;
-
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
-
-  PERFORM cartodb.cdb_disable_ddl_hooks();
-
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
-
-  PERFORM cartodb.cdb_enable_ddl_hooks();
-
-  -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
-
-END; $$;
--- }
-
--- Column drop
--- {
-CREATE OR REPLACE FUNCTION cartodb.cdb_handle_drop_column ()
-RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
-DECLARE
-  event_info RECORD;
-  rel RECORD;
-  rel_namespace TEXT;
-BEGIN
-  event_info := schema_triggers.get_column_drop_eventinfo();
-
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
-
-  RAISE DEBUG 'Column % drop by % (superuser? %) in relation % of kind %',
-	 (event_info.old).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
-
-  -- We're only interested in real relations
-  IF rel.relkind != 'r' THEN RETURN; END IF;
-
-  -- We don't want to react to drops triggered by superuser,
-  IF current_setting('is_superuser') = 'on' THEN
-    RAISE DEBUG 'no ddl trigger for superuser';
-    RETURN;
-  END IF;
-
-  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
-
-  PERFORM cartodb.cdb_disable_ddl_hooks();
-
-  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
-
-  PERFORM cartodb.cdb_enable_ddl_hooks();
-
-  -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
-
-END; $$;
--- }
-
--- Column add
--- {
-CREATE OR REPLACE FUNCTION cartodb.cdb_handle_add_column ()
-RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
-DECLARE
-  event_info RECORD;
-  rel RECORD;
-BEGIN
-  event_info := schema_triggers.get_column_add_eventinfo();
-
-  SELECT oid,* FROM pg_class WHERE oid = event_info.relation INTO rel;
-
-  RAISE DEBUG 'Column % added by % (superuser? %) in relation % of kind %',
-	 (event_info.new).attname, current_user, current_setting('is_superuser'), event_info.relation::regclass, rel.relkind;
-
-  -- We're only interested in real relations
-  IF rel.relkind != 'r' THEN RETURN; END IF;
-
-  -- We don't want to react to drops triggered by superuser,
-  IF current_setting('is_superuser') = 'on' THEN
-    RAISE DEBUG 'no ddl trigger for superuser';
-    RETURN;
-  END IF;
-
-  -- update CDB_TableMetadata.updated_at (should invalidate varnish)
-  UPDATE cartodb.CDB_TableMetadata SET updated_at = NOW()
-  WHERE tabname = event_info.relation; 
-
-END; $$;
--- }
-
-CREATE OR REPLACE FUNCTION cartodb.cdb_disable_ddl_hooks() returns void AS $$
- DROP EVENT TRIGGER IF EXISTS cdb_on_relation_create;
- DROP EVENT TRIGGER IF EXISTS cdb_on_relation_drop;
- DROP EVENT TRIGGER IF EXISTS cdb_on_alter_column;
- DROP EVENT TRIGGER IF EXISTS cdb_on_drop_column;
- DROP EVENT TRIGGER IF EXISTS cdb_on_add_column;
-$$ LANGUAGE sql;
-
-CREATE OR REPLACE FUNCTION cartodb.cdb_enable_ddl_hooks() returns void AS $$
- SELECT cartodb.cdb_disable_ddl_hooks();
- CREATE EVENT TRIGGER cdb_on_relation_create
-    ON "relation_create" EXECUTE PROCEDURE cartodb.cdb_handle_create_table();
- CREATE EVENT TRIGGER cdb_on_relation_drop
-    ON "relation_drop" EXECUTE PROCEDURE cartodb.cdb_handle_drop_table();
- CREATE EVENT TRIGGER cdb_on_alter_column
-    ON "column_alter" EXECUTE PROCEDURE cartodb.cdb_handle_alter_column();
- CREATE EVENT TRIGGER cdb_on_drop_column
-    ON "column_drop" EXECUTE PROCEDURE cartodb.cdb_handle_drop_column();
- CREATE EVENT TRIGGER cdb_on_add_column
-    ON "column_add" EXECUTE PROCEDURE cartodb.cdb_handle_add_column();
-$$ LANGUAGE sql;
-
--- Do not enable hooks by default
---SELECT cartodb.cdb_enable_ddl_hooks();

scripts-available/CDB_DateToNumber.sql

@@ -13,3 +13,19 @@ RETURN output;
 END;
 $$
 LANGUAGE 'plpgsql' STABLE STRICT;
+
+-- Convert timestamp with time zone to double precision
+--
+CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp with time zone)
+RETURNS double precision AS $$
+DECLARE output double precision;
+BEGIN
+    BEGIN
+        SELECT extract (EPOCH FROM input) INTO output;
+    EXCEPTION WHEN OTHERS THEN
+        RETURN NULL;
+    END;
+RETURN output;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE STRICT;

scripts-available/CDB_GreatCircle.sql

@@ -0,0 +1,26 @@
+-- Great circle point-to-point routes, based on:
+--   http://blog.cartodb.com/jets-and-datelines/
+--
+CREATE OR REPLACE FUNCTION CDB_GreatCircle(start_point geometry, end_point geometry, max_segment_length NUMERIC DEFAULT 100000)
+RETURNS geometry AS $$
+DECLARE
+  line geometry;
+BEGIN
+  line = ST_Segmentize(
+    ST_Makeline(
+      start_point,
+      end_point
+    )::geography,
+    max_segment_length
+  )::geometry;
+
+  IF ST_XMax(line) - ST_XMin(line) > 180 THEN
+    line = ST_Difference(
+      ST_Shift_Longitude(line),
+			ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001)
+		);
+  END IF;
+RETURN line;
+END;
+$$
+LANGUAGE 'plpgsql';

scripts-available/CDB_Groups.sql

@@ -0,0 +1,252 @@
+----------------------------------
+-- GROUP MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by org admin. See CDB_Organization_AddAdmin.
+----------------------------------
+
+-- Creates a new group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_CreateGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('CREATE ROLE %I NOLOGIN;', group_role);
+    PERFORM cartodb._CDB_Group_CreateGroup_API(group_name, group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Drops group and everything that role owns
+-- TODO: LIMITATION: in order to drop a role all its owned objects must be dropped before.
+-- Right now this is done with DROP OWNED, which can only be done by a superadmin.
+-- Not even the role creator can drop the role and the objects it owns.
+-- All group owned objects by the group are permissions.
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_DropGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('DROP OWNED BY %I', group_role);
+    EXECUTE format('DROP ROLE IF EXISTS %I', group_role);
+    PERFORM cartodb._CDB_Group_DropGroup_API(group_name);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Renames a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RenameGroup(old_group_name text, new_group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    old_group_role TEXT;
+    new_group_role TEXT;
+BEGIN
+    old_group_role = cartodb._CDB_Group_GroupRole(old_group_name);
+    new_group_role = cartodb._CDB_Group_GroupRole(new_group_name);
+    EXECUTE format('ALTER ROLE %I RENAME TO %I', old_group_role, new_group_role);
+    PERFORM cartodb._CDB_Group_RenameGroup_API(old_group_name, new_group_name, new_group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Adds users to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_AddUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      IF(group_role IS NULL OR user_role IS NULL)
+      THEN
+        RAISE EXCEPTION 'Group role (%) and user role (%) must be already existing', group_role, user_role;
+      END IF;
+      EXECUTE format('GRANT %I TO %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_AddUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Removes users from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RemoveUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      EXECUTE format('REVOKE %I FROM %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_RemoveUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+----------------------------------
+-- TABLE MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by table owners.
+----------------------------------
+
+-- Grants table read permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantRead(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantRead(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantRead(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT ON TABLE %I.%I TO %I', username, table_name, group_role );
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'r');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Grants table write permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantReadWrite(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE %I.%I TO %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, true);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'w');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Granting and revoking permissions on sequences
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_TableSequences_Permission(group_name text, username text, table_name text, do_grant bool)
+    RETURNS VOID AS $$
+DECLARE
+    column_name TEXT;
+    sequence_name TEXT;
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    FOR column_name IN EXECUTE 'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = current_database() AND TABLE_SCHEMA = $1 AND TABLE_NAME = $2 AND COLUMN_DEFAULT LIKE ''nextval%''' USING username, table_name
+    LOOP
+        EXECUTE format('SELECT PG_GET_SERIAL_SEQUENCE(''%I.%I'', ''%I'')', username, table_name, column_name) INTO sequence_name;
+        IF sequence_name IS NOT NULL THEN
+          IF do_grant THEN
+            -- Here %s is needed since sequence_name has quotes
+            EXECUTE format('GRANT USAGE, SELECT, UPDATE ON SEQUENCE %s TO %I', sequence_name, group_role);
+          ELSE
+            EXECUTE format('REVOKE ALL ON SEQUENCE %s FROM %I', sequence_name, group_role);
+          END IF;
+        END IF;
+    END LOOP;
+    RETURN;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Revokes all permissions on a table from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_RevokeAll(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_RevokeAll(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAll(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('REVOKE ALL ON TABLE %I.%I FROM %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, false);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name, username, table_name);
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-----------------------
+-- Helper functions
+-----------------------
+-- Given a group name returns a role. group_name must be a valid PostgreSQL idenfifier. See http://www.postgresql.org/docs/9.2/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_GroupRole(group_name text)
+    RETURNS TEXT AS $$
+DECLARE
+    group_role TEXT;
+    prefix TEXT;
+    max_length constant INTEGER := 63;
+BEGIN
+    prefix = format('%s_g_', cartodb._CDB_Group_ShortDatabaseName());
+    group_role := format('%s%s', prefix, group_name);
+    IF LENGTH(group_role) > max_length
+    THEN
+        RAISE EXCEPTION 'Group name must be shorter. It can''t have more than % characters, but it is longer (%): %', max_length - LENGTH(prefix), length(group_name), group_name;
+    END IF;
+    RETURN group_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Returns the first owner of the schema matching username. Organization user schemas must have one only owner.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_User_RoleFromUsername(username text)
+    RETURNS TEXT AS $$
+DECLARE
+    user_role TEXT;
+BEGIN
+    -- This was preferred, but non-superadmins won't get results
+    -- SELECT SCHEMA_OWNER FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = $1 LIMIT 1'
+    SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = username INTO user_role;
+    RETURN user_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Database names are too long, we need a shorter version for composing role names
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_ShortDatabaseName()
+    RETURNS TEXT AS $$
+DECLARE
+    short_database_name TEXT;
+BEGIN
+    SELECT md5(current_database()) INTO short_database_name;
+    RETURN short_database_name;
+END
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_Groups_API.sql

@@ -0,0 +1,195 @@
+----------------------------------
+-- GROUP METADATA API FUNCTIONS
+--
+-- Meant to be used by CDB_Group_* functions to sync data with the editor.
+-- Requires configuration parameter. Example: SELECT cartodb.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
+----------------------------------
+
+-- TODO: delete this development cleanup before final merge
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_AddMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_RemoveMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_AddMember_API(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_RemoveMember_API(group_name text, username text);
+
+-- Sends the create group request
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_CreateGroup_API(group_name text, group_role text)
+    RETURNS VOID AS
+$$
+    import string
+
+    url = '/api/v1/databases/{0}/groups'
+    body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_DropGroup_API(group_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_AddUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantPermission_API(group_name text, username text, table_name text, access text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    body = '{ "access": "%s" }' % access
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name text, username text, table_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE TYPE _CDB_Group_API_Params AS (
+    host text,
+    port int,
+    timeout int,
+    auth text
+);
+
+-- This must be explicitally extracted because "composite types are currently not supported".
+-- See http://www.postgresql.org/docs/9.3/static/plpython-database.html.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Conf()
+    RETURNS _CDB_Group_API_Params AS
+$$
+    conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('groups_api') conf")[0]['conf']
+    if conf is None:
+      return None
+    else:
+      import json
+      params = json.loads(conf)
+      auth = 'Basic %s' % plpy.execute("SELECT cartodb._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
+      return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Auth(username text, password text)
+    RETURNS TEXT AS
+$$
+    import base64
+    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+-- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
+    RETURNS int AS
+$$
+    import httplib
+
+    params = plpy.execute("select c.host, c.port, c.timeout, c.auth from cartodb._CDB_Group_API_Conf() c;")[0]
+    if params['host'] is None:
+      return None
+
+    headers = { 'Authorization': params['auth'], 'Content-Type': 'application/json', 'X-Forwarded-Proto': 'https' }
+
+    retry = 3
+
+    last_err = None
+    while retry > 0:
+      try:
+        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        database_name = plpy.execute("select current_database();")[0]['current_database']
+        client.request(method, url.format(database_name), body, headers)
+        response = client.getresponse()
+        assert response.status in valid_return_codes
+        return response.status
+      except Exception as err:
+        retry -= 1
+        last_err = err
+        plpy.warning('Retrying after: ' + str(err))
+        client = SD['groups_api_client'] = None
+
+    if last_err is not None:
+      plpy.error('Fatal Group API error: ' + str(last_err))
+      raise last_err
+
+    return None
+$$ LANGUAGE 'plpythonu' VOLATILE;
+revoke all on function cartodb._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_Helper.sql

@@ -0,0 +1,156 @@
+-- UTF8 safe and length aware. Find a unique identifier with a given prefix
+-- and/or suffix and withing a schema. If a schema is not specified, the identifier
+-- is guaranteed to be unique for all schemas.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Identifier(prefix TEXT, relname TEXT, suffix TEXT, schema TEXT DEFAULT NULL)
+RETURNS TEXT
+AS $$
+DECLARE
+  maxlen CONSTANT INTEGER := 63;
+
+  rec RECORD;
+  usedspace INTEGER;
+  ident TEXT;
+  origident TEXT;
+  candrelname TEXT;
+
+  i INTEGER;
+BEGIN
+  -- Accounts for the _XX incremental suffix in case the identifier is taken
+  usedspace := 3;
+  usedspace := usedspace + coalesce(octet_length(prefix), 0);
+  usedspace := usedspace + coalesce(octet_length(suffix), 0);
+
+  candrelname := _CDB_Octet_Truncate(relname, maxlen - usedspace);
+
+  IF candrelname = '' THEN
+    PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Identifier');
+  END IF;
+
+  ident := coalesce(prefix, '') || candrelname || coalesce(suffix, '');
+
+  i := 0;
+  origident := ident;
+
+  WHILE i < 100 LOOP
+    IF schema IS NOT NULL THEN
+      SELECT c.relname, n.nspname
+      INTO rec
+      FROM pg_class c
+      JOIN pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.relname = ident
+      AND n.nspname = schema;
+    ELSE
+      SELECT c.relname, n.nspname
+      INTO rec
+      FROM pg_class c
+      JOIN pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.relname = ident;
+    END IF;
+
+    IF NOT FOUND THEN
+      RETURN ident;
+    END IF;
+
+    ident := origident || '_' || i;
+    i := i + 1;
+  END LOOP;
+
+  PERFORM _CDB_Error('looping too far', '_CDB_Unique_Identifier');
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- UTF8 safe and length aware. Find a unique identifier for a column with a given prefix
+-- and/or suffix based on colname and within a relation specified via reloid.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Unique_Column_Identifier(prefix TEXT, colname TEXT, suffix TEXT, reloid REGCLASS)
+RETURNS TEXT
+AS $$
+DECLARE
+  maxlen CONSTANT INTEGER := 63;
+
+  rec RECORD;
+  candcolname TEXT;
+  usedspace INTEGER;
+  ident TEXT;
+  origident TEXT;
+
+  i INTEGER;
+BEGIN
+  -- Accounts for the _XX incremental suffix in case the identifier is taken
+  usedspace := 3;
+  usedspace := usedspace + coalesce(octet_length(prefix), 0);
+  usedspace := usedspace + coalesce(octet_length(suffix), 0);
+
+  candcolname := _CDB_Octet_Truncate(colname, maxlen - usedspace);
+
+  IF candcolname = '' THEN
+    PERFORM _CDB_Error('prefixes are to long to generate a valid identifier', '_CDB_Unique_Column_Identifier');
+  END IF;
+
+  ident := coalesce(prefix, '') || candcolname || coalesce(suffix, '');
+
+  i := 0;
+  origident := ident;
+
+  WHILE i < 100 LOOP
+    SELECT a.attname
+    INTO rec
+    FROM pg_class c
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE NOT a.attisdropped
+    AND a.attnum > 0
+    AND c.oid = reloid
+    AND a.attname = ident;
+
+    IF NOT FOUND THEN
+      RETURN ident;
+    END IF;
+
+    ident := origident || '_' || i;
+    i := i + 1;
+  END LOOP;
+
+  PERFORM _CDB_Error('looping too far', '_CDB_Unique_Column_Identifier');
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+-- Truncates a given string to a max_octets octets taking care
+-- not to leave characters in half. UTF8 safe.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Octet_Truncate(string TEXT, max_octets INTEGER)
+RETURNS TEXT
+AS $$
+DECLARE
+  extcharlen CONSTANT INTEGER := octet_length('ñ');
+
+  expected INTEGER;
+  examined INTEGER;
+  strlen INTEGER;
+
+  i INTEGER;
+BEGIN
+
+  IF max_octets <= 0 THEN
+    RETURN '';
+  ELSIF max_octets >= octet_length(string) THEN
+    RETURN string;
+  END IF;
+
+  strlen := char_length(string);
+
+  expected := char_length(string);
+  examined := octet_length(string);
+
+  IF expected = examined THEN
+    RETURN left(string, max_octets);
+  END IF;
+
+  i := max_octets / extcharlen;
+
+  WHILE octet_length(left(string, i)) <= max_octets LOOP
+    i := i + 1;
+  END LOOP;
+
+  RETURN left(string, (i - 1));
+END;
+$$ LANGUAGE 'plpgsql';

scripts-available/CDB_LatLng.sql

@@ -8,16 +8,12 @@
 --
 
 CREATE OR REPLACE FUNCTION CDB_LatLng (lat NUMERIC, lng NUMERIC) RETURNS geometry as $$ 
-BEGIN 
     -- this function is silly
-    RETURN ST_SetSRID(ST_MakePoint(lng,lat),4326); 
-END; 
-$$ language plpgsql IMMUTABLE;
+    SELECT ST_SetSRID(ST_MakePoint(lng,lat),4326);
+$$ language SQL IMMUTABLE;
 
 CREATE OR REPLACE FUNCTION CDB_LatLng (lat FLOAT8, lng FLOAT8) RETURNS geometry as $$ 
-BEGIN 
     -- this function is silly
-    RETURN ST_SetSRID(ST_MakePoint(lng,lat),4326); 
-END; 
-$$ language plpgsql IMMUTABLE;
+    SELECT ST_SetSRID(ST_MakePoint(lng,lat),4326);
+$$ language SQL IMMUTABLE;
 

scripts-available/CDB_Organizations.sql

@@ -1,14 +1,16 @@
 CREATE OR REPLACE
 FUNCTION cartodb.CDB_Organization_Member_Group_Role_Member_Name()
     RETURNS TEXT
-AS 'SELECT ''cdb_org_member''::text || ''_'' || md5(current_database());'
+AS $$
+    SELECT 'cdb_org_member'::text || '_' || md5(current_database());
+$$
 LANGUAGE SQL IMMUTABLE;
 
 DO LANGUAGE 'plpgsql' $$
 DECLARE
     cdb_org_member_role_name TEXT;
 BEGIN
-    cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
+  cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
   IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_member_role_name )
   THEN
     EXECUTE 'CREATE ROLE "' || cdb_org_member_role_name || '" NOLOGIN;';
@@ -25,6 +27,60 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
+-------------------------------------------------------------------------------
+-- Administrator
+-------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Organization_Admin_Role_Name()
+    RETURNS TEXT
+AS $$
+    SELECT current_database() || '_a'::text;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+-- Administrator role creation on extension install
+DO LANGUAGE 'plpgsql' $$
+DECLARE
+    cdb_org_admin_role_name TEXT;
+BEGIN
+    cdb_org_admin_role_name := cartodb._CDB_Organization_Admin_Role_Name();
+    IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_admin_role_name )
+    THEN
+        EXECUTE format('CREATE ROLE %I CREATEROLE NOLOGIN;', cdb_org_admin_role_name);
+    END IF;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_AddAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('GRANT %I TO %I WITH ADMIN OPTION', cdb_admin_role, cdb_user_role);
+    -- CREATEROLE is not inherited, and is needed for user creation
+    EXECUTE format('ALTER ROLE %I CREATEROLE', cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_RemoveAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('ALTER ROLE %I NOCREATEROLE', cdb_user_role);
+    EXECUTE format('REVOKE %I FROM %I', cdb_admin_role, cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
 
 -------------------------------------------------------------------------------
 -- Sharing tables

scripts-available/CDB_Overviews.sql

@@ -0,0 +1,687 @@
+-- security definer
+
+-- Pattern that can be used to detect overview tables and Extract
+-- the intended zoom level from the table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '\A_vovw_(\d+)_';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- substring(tablename from _CDB_OverviewTableDiscriminator())
+
+
+-- Pattern matched by the overview tables of a given base table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN _CDB_OverviewTableDiscriminator() || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
+
+-- Name of an overview table, given the base table name and the Z level
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '_vovw_' || z::text || '_' || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Condition to check if a tabla is an overview table of some base table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Extract the Z level from an overview table name
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Name of the base table corresponding to an overview table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
+      RETURN overview_table;
+    ELSE
+      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
+    END IF;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+
+-- Remove a dataset's existing  overview tables.
+-- Scope: public
+-- Parameters:
+--   reloid: oid of the table.
+CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
+RETURNS void
+AS $$
+DECLARE
+    row     record;
+BEGIN
+    FOR row IN
+        SELECT * FROM CDB_Overviews(reloid)
+    LOOP
+        EXECUTE Format('DROP TABLE %s;', row.overview_table);
+        RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table;
+    END LOOP;
+END;
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+
+
+-- Return existing overviews (if any) for a given dataset table
+-- Scope: public
+-- Parameters
+--   reloid: oid of the input table.
+-- Return relation of overviews for the table with
+-- the base table oid,
+-- z level of the overview and overview table oid, ordered by z.
+CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
+RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
+AS $$
+  -- FIXME: this will fail if the overview tables
+  -- require a explicit schema name
+  -- possible solutions: return table names as text instead of regclass
+  -- or add schema of reloid before casting to regclass
+  SELECT
+    reloid AS base_table,
+    _CDB_OverviewTableZ(cdb_usertables) AS z,
+    cdb_usertables::regclass AS overview_table
+    FROM CDB_UserTables()
+    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
+    ORDER BY z;
+$$ LANGUAGE SQL;
+
+-- Return existing overviews (if any) for multiple dataset tables.
+-- Scope: public
+-- Parameters
+--   tables: Array of input tables oids
+-- Return relation of overviews for the table with
+-- the base table oid,
+-- z level of the overview and overview table oid, ordered by z.
+-- Note: CDB_Overviews can be applied to the result of CDB_QueryTablesText
+-- to obtain the overviews applicable to a query.
+CREATE OR REPLACE FUNCTION CDB_Overviews(tables regclass[])
+RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
+AS $$
+  SELECT
+    base_table::regclass AS base_table,
+    _CDB_OverviewTableZ(cdb_usertables) AS z,
+    cdb_usertables::regclass AS overview_table
+    FROM
+      CDB_UserTables(), unnest(tables) base_table
+    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
+    ORDER BY base_table, z;
+$$ LANGUAGE SQL;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Calculate the estimated extent of a cartodbfy'ed table.
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table.
+-- Return value A box2d extent in 3857.
+CREATE OR REPLACE FUNCTION _cdb_estimated_extent(reloid REGCLASS)
+RETURNS box2d
+AS $$
+  DECLARE
+    ext box2d;
+    ext_query text;
+    table_id record;
+  BEGIN
+
+    SELECT n.nspname AS schema_name, c.relname table_name INTO STRICT table_id
+      FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
+
+    ext_query = format(
+      'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
+      table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
+    );
+
+    BEGIN
+      EXECUTE ext_query INTO ext;
+      EXCEPTION
+        -- This is the typical ERROR: stats for "mytable" do not exist
+        WHEN internal_error THEN
+          -- Get stats and execute again
+          EXECUTE format('ANALYZE %1$I', reloid);
+          EXECUTE ext_query INTO ext;
+    END;
+
+    RETURN ext;
+  END;
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Determine the max feature density of a given dataset.
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+--   nz: number of zoom levels to consider from z0 upward.
+-- Return value: feature density (num_features / webmercator_squared_meters).
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density(reloid REGCLASS, nz integer)
+RETURNS FLOAT8
+AS $$
+  DECLARE
+    fd FLOAT8;
+    min_features TEXT;
+    n integer = 4;
+    c FLOAT8;
+  BEGIN
+  -- TODO: for small total count or extents we could just:
+  -- EXECUTE 'SELECT Count(*)/ST_Area(ST_Extent(the_geom_webmercator)) FROM ' || reloid::text || ';' INTO fd;
+
+  -- min_features is a SQL subexpression which can depend on z and represents
+  -- the minimum number of features to recursively consider a tile.
+  -- We can either use a fixed minimum number of features per tile
+  -- or a minimum feature density by dividing the number of features by
+  -- the area of tiles at level Z: c*c*power(2, -2*z)
+  -- with c = CDB_XYZ_Resolution(-8) (earth circumference)
+  min_features = '500';
+  SELECT CDB_XYZ_Resolution(-8) INTO c;
+
+  -- We first compute a set of *seed* tiles, of the minimum Z level, z0, such that
+  -- they cover the extent of the table and we have at least n of them in each
+  -- linear dimension (i.e. at least n*n tiles cover the extent).
+  -- We compute the number of features in these tiles, and recursively in
+  -- subtiles up to level z0 + nz. Then we compute the maximum of the feature
+  -- density (per tile area in webmercator squared meters) for all the
+  -- considered tiles.
+  EXECUTE Format('
+    WITH RECURSIVE t(x, y, z, e) AS (
+      WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
+      base AS (
+        SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
+        FROM ext
+      ),
+      lim AS (
+        SELECT
+          FLOOR((ST_XMin(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x0,
+          FLOOR((ST_XMax(ext.g)+CDB_XYZ_Resolution(0)*128)/(CDB_XYZ_Resolution(base.z)*256))::integer x1,
+          FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMin(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y1,
+          FLOOR((CDB_XYZ_Resolution(0)*128-ST_YMax(ext.g))/(CDB_XYZ_Resolution(base.z)*256))::integer y0
+        FROM ext, base
+      ),
+      seed AS (
+        SELECT xt, yt, base.z, (
+          SELECT count(*) FROM %1$s
+            WHERE the_geom_webmercator && CDB_XYZ_Extent(xt, yt, base.z)
+        ) e
+        FROM base, lim, generate_series(lim.x0, lim.x1) xt, generate_series(lim.y0, lim.y1) yt
+      )
+      SELECT * from seed
+      UNION ALL
+      SELECT x*2 + xx, y*2 + yy, t.z+1, (
+        SELECT count(*) FROM %1$s
+          WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
+      )
+      FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
+      WHERE t.e > %2$s AND t.z < (base.z + %3$s)
+    )
+    SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
+  ', reloid::text, min_features, nz, n, c, reloid::oid)
+  INTO fd;
+  RETURN fd;
+  END
+$$ LANGUAGE PLPGSQL STABLE;
+
+-- Experimental default strategy to assign a reference base Z level
+-- to a cartodbfied table. The resulting Z level represents the
+-- minimum scale level at which the table data can be rendered
+-- without overcrowded results or loss of detail.
+-- Parameters:
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+-- Return value: Z level as an integer
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
+RETURNS INTEGER
+AS $$
+  DECLARE
+    lim FLOAT8 := 500; -- TODO: determine/parameterize this
+    nz integer := 4;
+    fd FLOAT8;
+    c FLOAT8;
+  BEGIN
+    -- Compute fd as an estimation of the (maximum) number
+    -- of features per unit of tile area (in webmercator squared meters)
+    SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
+    -- lim maximum number of (desiderable) features per tile
+    -- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
+    -- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
+    -- => fd*ta(z) if the average number of features per tile at level z
+    -- find minimum z so that fd*ta(z) <= lim
+    -- compute a rough 'feature density' value
+    SELECT CDB_XYZ_Resolution(-8) INTO c;
+    RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
+  END;
+$$ LANGUAGE PLPGSQL STABLE;
+
+-- Overview table name for a given Z level and base dataset or overview table
+-- Scope: private.
+-- Parameters:
+--   ref reference table (can be the base table of the dataset or an existing
+--   overview) from which the overview is being generated.
+--   ref_z Z level of the reference table
+--   overview_z Z level of the overview to be named, must be smaller than ref_z
+-- Return value: the name to be used for the overview. The name is always
+-- unqualified (does not include a schema name).
+CREATE OR REPLACE FUNCTION _CDB_Overview_Name(ref REGCLASS, ref_z INTEGER, overview_z INTEGER)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+    base TEXT;
+    suffix TEXT;
+    is_overview BOOLEAN;
+  BEGIN
+    SELECT * FROM _cdb_split_table_name(ref) INTO schema_name, base;
+    SELECT _CDB_OverviewBaseTableName(base) INTO base;
+    RETURN _CDB_OverviewTableName(base, overview_z);
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Sampling reduction method.
+-- Valid for any kind of geometry.
+-- Scope: private.
+--   reloid original table (can be the base table of the dataset or an existing
+--   overview) from which the overview is being generated.
+--   ref_z Z level assigned to the original table
+--   overview_z Z level of the overview to be generated, must be smaller than ref_z
+-- Return value: Name of the generated overview table
+CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    overview_rel TEXT;
+    fraction FLOAT8;
+    base_name TEXT;
+    class_info RECORD;
+    num_samples INTEGER;
+  BEGIN
+    overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+    fraction := power(2, 2*(overview_z - ref_z));
+
+    -- FIXME: handle schema name for overview_rel if reloid requires it
+    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+
+    -- Estimate number of rows
+    SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
+      WHERE oid = reloid::oid;
+
+    IF class_info.relpages < 2 OR fraction > 0.5 THEN
+      -- We'll avoid possible CDB_RandomTids problems
+      EXECUTE Format('
+        CREATE TABLE %I AS SELECT * FROM %s WHERE random() < %s;
+      ', overview_rel, reloid, fraction);
+    ELSE
+      num_samples := ceil(class_info.reltuples*fraction);
+      EXECUTE Format('
+        CREATE TABLE %1$I AS SELECT * FROM %2$s
+          WHERE ctid = ANY (
+            ARRAY[
+              (SELECT CDB_RandomTids(''%2$s'', %3$s))
+            ]
+          );
+      ', overview_rel, reloid, num_samples);
+    END IF;
+
+    RETURN overview_rel;
+  END;
+$$ LANGUAGE PLPGSQL;
+
+-- Register new overview table (post-creation chores)
+-- Scope: private
+-- Parameters:
+--   dataset: oid of the input dataset table,  It must be a cartodbfy'ed table.
+--   overview_table: oid of the overview table to be registered.
+--   overview_z: intended Z level for the overview table
+-- This function is declared SECURITY DEFINER so it executes with the privileges
+-- of the function creator to have a chance to alter the privileges of the
+-- overview table to match those of the dataset. It will only perform any change
+-- if the overview table belgons to the same scheme as the dataset and it
+-- matches the scheme naming for overview tables.
+CREATE OR REPLACE FUNCTION _CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
+RETURNS VOID
+AS $$
+  DECLARE
+    sql TEXT;
+    table_owner TEXT;
+    dataset_scheme TEXT;
+    dataset_name TEXT;
+    overview_scheme TEXT;
+    overview_name TEXT;
+  BEGIN
+    -- This function will only register a table as an overview table if it matches
+    -- the overviews naming scheme for the dataset and z level and the table belongs
+    -- to the same scheme as the the dataset
+    SELECT * FROM _cdb_split_table_name(dataset) INTO dataset_scheme, dataset_name;
+    SELECT * FROM _cdb_split_table_name(overview_table) INTO overview_scheme, overview_name;
+    IF dataset_scheme = overview_scheme AND
+       overview_name = _CDB_OverviewTableName(dataset_name, overview_z) THEN
+
+      -- preserve the owner of the base table
+      SELECT u.usename
+        FROM pg_catalog.pg_class c JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
+        WHERE c.relname = dataset::text
+        INTO table_owner;
+      EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
+
+      -- preserve the table privileges
+      UPDATE pg_class c_to
+        SET  relacl = c_from.relacl
+        FROM  pg_class c_from
+        WHERE c_from.oid  = dataset
+        AND   c_to.oid    = overview_table;
+
+      PERFORM _CDB_Add_Indexes(overview_table);
+
+      -- TODO: If metadata about existing overviews is to be stored
+      -- it should be done here (CDB_Overviews would consume such metadata)
+    END IF;
+  END
+$$ LANGUAGE PLPGSQL SECURITY DEFINER;
+
+-- Dataset attributes (column names other than the
+-- CartoDB primary key and geometry columns) which should be aggregated
+-- in aggregated overviews.
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+-- Return value: set of attribute names
+CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes(reloid REGCLASS)
+RETURNS SETOF information_schema.sql_identifier
+AS $$
+  SELECT c FROM CDB_ColumnNames(reloid) c, _CDB_Columns() cdb
+    WHERE c NOT IN (
+      cdb.pkey, cdb.geomcol, cdb.mercgeomcol
+    )
+$$ LANGUAGE SQL STABLE;
+
+-- List of dataset attributes to be aggregated in aggregated overview
+-- as a comma-separated SQL expression.
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+-- Return value: SQL subexpression as text
+CREATE OR REPLACE FUNCTION _CDB_Aggregable_Attributes_Expression(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+DECLARE
+  attr_list TEXT;
+BEGIN
+  SELECT string_agg(s.c, ',') FROM (
+    SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
+  ) AS s INTO attr_list;
+
+  RETURN attr_list;
+END
+$$ LANGUAGE PLPGSQL STABLE;
+
+-- SQL Aggregation expression for a datase attribute
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+--   column_name: column to be aggregated
+--   table_alias: (optional) table qualifier for the column to be aggregated
+-- Return SQL subexpression as text with aggregated attribute aliased
+-- with its original name.
+CREATE OR REPLACE FUNCTION _CDB_Attribute_Aggregation_Expression(reloid REGCLASS, column_name TEXT, table_alias TEXT DEFAULT '')
+RETURNS TEXT
+AS $$
+DECLARE
+  column_type TEXT;
+  qualified_column TEXT;
+BEGIN
+  IF table_alias <> '' THEN
+    qualified_column := Format('%I.%I', table_alias, column_name);
+  ELSE
+    qualified_column := Format('%I', column_name);
+  END IF;
+
+  column_type := CDB_ColumnType(reloid, column_name);
+
+  CASE column_type
+  WHEN 'double precision', 'real', 'integer', 'bigint' THEN
+    RETURN Format('AVG(%s)::' || column_type, qualified_column);
+  WHEN 'text' THEN
+    -- TODO: we could define a new aggregate function that returns distinct
+    -- separated values with a limit, adding ellipsis if more values existed
+    -- e.g. with '/' as separator and a limit of three:
+    --     'A', 'B', 'A', 'C', 'D' => 'A/B/C/...'
+    -- Other ideas: if value is unique then use it, otherwise use something
+    -- like '*' or '(varies)' or '(multiple values)', or NULL
+    -- Using 'string_agg(' || qualified_column || ',''/'')'
+    -- here causes
+    RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+  ELSE
+    RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+  END CASE;
+END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- List of dataset aggregated attributes as a comma-separated SQL expression.
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+--   table_alias: (optional) table qualifier for the columns to be aggregated
+-- Return value: SQL subexpression as text
+CREATE OR REPLACE FUNCTION _CDB_Aggregated_Attributes_Expression(reloid REGCLASS, table_alias TEXT DEFAULT '')
+RETURNS TEXT
+AS $$
+DECLARE
+  attr_list TEXT;
+BEGIN
+  SELECT string_agg(_CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
+  FROM (
+    SELECT * FROM _CDB_Aggregable_Attributes(reloid) c
+  ) AS s INTO attr_list;
+
+  RETURN attr_list;
+END
+$$ LANGUAGE PLPGSQL STABLE;
+
+-- Array of geometry types detected in a cartodbfied table
+-- For effciency only look at a limited number of rwos.
+-- Parameters
+--   reloid: oid of the input table. It must be a cartodbfy'ed table.
+-- Return value: array of geometry type names
+CREATE OR REPLACE FUNCTION _CDB_GeometryTypes(reloid REGCLASS)
+RETURNS TEXT[]
+AS $$
+DECLARE
+  gtypes TEXT[];
+BEGIN
+  EXECUTE Format('
+    SELECT array_agg(DISTINCT ST_GeometryType(the_geom)) FROM (
+      SELECT the_geom FROM %s
+        WHERE (the_geom is not null) LIMIT 10
+    ) as geom_types
+  ', reloid)
+  INTO gtypes;
+  RETURN gtypes;
+END
+$$ LANGUAGE PLPGSQL STABLE;
+
+-- Experimental Overview reduction method for point datasets.
+-- It clusters the points using a grid, then aggregates the point in each
+-- cluster into a point at the centroid of the clustered records.
+-- Scope: private.
+-- Parameters:
+--   reloid original table (can be the base table of the dataset or an existing
+--   overview) from which the overview is being generated.
+--   ref_z Z level assigned to the original table
+--   overview_z Z level of the overview to be generated, must be smaller than ref_z
+-- Return value: Name of the generated overview table
+CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    overview_rel TEXT;
+    reduction FLOAT8;
+    base_name TEXT;
+    grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
+    grid_m FLOAT8;
+    aggr_attributes TEXT;
+    attributes TEXT;
+    columns TEXT;
+    gtypes TEXT[];
+  BEGIN
+    SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
+    IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+      RETURN 'x';
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- compute grid cell size using the overview_z dimension...
+    SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
+
+    attributes := _CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- compute the resulting columns in the same order as in the base table
+    -- cartodb_id,
+    -- ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom,
+    -- ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator
+    -- %4$s
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        WHEN 'the_geom' THEN
+          'ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom'
+        WHEN 'the_geom_webmercator' THEN
+           'ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator'
+        ELSE c
+        END AS column
+        FROM CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    -- FIXME: handle schema name for overview_rel if reloid requires it
+    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and selecte the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+      CREATE TABLE %3$I AS
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             SUM(ST_X(f.the_geom_webmercator)) AS sx,
+             SUM(ST_Y(f.the_geom_webmercator)) AS sy,
+             Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s f
+          GROUP BY gx, gy
+         )
+         SELECT %6$s FROM clusters
+    ', reloid::text, grid_m, overview_rel, attributes, aggr_attributes, columns);
+
+    RETURN overview_rel;
+  END;
+$$ LANGUAGE PLPGSQL;
+
+-- Create overview tables for a dataset.
+-- Scope: public
+-- Parameters:
+--   reloid: oid of the input table. It must be a cartodbfy'ed table with
+--           vector features.
+--   refscale_strategy: function that computes the reference Z of the dataset
+--   reduce_strategy: function that generates overviews from a base table
+--                    or higher level overview. The overview tables
+--                    created by the strategy must have the same columns
+--                    as the base table and in the same order.
+-- Return value: Array with the names of the generated overview tables
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(
+  reloid REGCLASS,
+  refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc,
+  reduce_strategy   regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc
+)
+RETURNS text[]
+AS $$
+DECLARE
+  ref_z integer;
+  overviews_z integer[];
+  base_z integer;
+  base_rel REGCLASS;
+  overview_z integer;
+  overview_tables REGCLASS[];
+  overviews_step integer := 1;
+BEGIN
+  -- Determine the referece zoom level
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
+
+  -- Determine overlay zoom levels
+  -- TODO: should be handled by the refscale_strategy?
+  overview_z := ref_z - 1;
+  WHILE overview_z >= 0 LOOP
+    SELECT array_append(overviews_z, overview_z) INTO overviews_z;
+    overview_z := overview_z - overviews_step;
+  END LOOP;
+
+  -- Create overlay tables
+  base_z := ref_z;
+  base_rel := reloid;
+  FOREACH overview_z IN ARRAY overviews_z LOOP
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
+    IF base_rel IS NULL THEN
+      EXIT;
+    END IF;
+    base_z := overview_z;
+    PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
+    SELECT array_append(overview_tables, base_rel) INTO overview_tables;
+  END LOOP;
+
+  RETURN overview_tables;
+END;
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;

scripts-available/CDB_Quota.sql

@@ -1,3 +1,19 @@
+CREATE OR REPLACE FUNCTION cartodb._CDB_total_relation_size(_schema_name TEXT, _table_name TEXT)
+RETURNS bigint AS
+$$
+DECLARE relation_size bigint := 0;
+BEGIN
+  BEGIN
+    SELECT pg_total_relation_size(format('"%s"."%s"', _schema_name, _table_name)) INTO relation_size;
+  EXCEPTION
+    WHEN undefined_table OR OTHERS THEN
+      RAISE NOTICE 'cartodb._CDB_total_relation_size(''%'', ''%'') caught error: % (%)', _schema_name, _table_name, SQLERRM, SQLSTATE;
+  END;
+  RETURN relation_size;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;
+
 -- Return the estimated size of user data. Used for quota checking.
 CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
 RETURNS bigint AS
@@ -24,7 +40,7 @@ BEGIN
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(pg_total_relation_size('"' || schema_name || '"."' || table_name || '"')))) table_size,
+    SELECT COALESCE(INT8(SUM(cartodb._CDB_total_relation_size(schema_name, table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1

scripts-available/CDB_Stats.sql

@@ -0,0 +1,47 @@
+--
+-- Calculate basic statistics of a given dataset
+--
+-- @param in_array A numeric array of numbers
+--
+-- Returns: statistical quantity chosen
+-- 
+-- References: http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm
+--
+
+-- Calculate kurtosis
+CREATE OR REPLACE FUNCTION CDB_Kurtosis ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    k numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 4)) / ( $2 * power($3, 4)) - 3
+             FROM (SELECT unnest($4) e ) x'
+    INTO k
+    USING a, c, s, in_array;
+
+    RETURN k;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Calculate skewness
+CREATE OR REPLACE FUNCTION CDB_Skewness ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    sk numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 * power($3, 3))
+             FROM (SELECT unnest($4) e ) x'
+    INTO sk
+    USING a, c, s, in_array;
+
+    RETURN sk;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_UserTables.sql

@@ -5,7 +5,7 @@
 --
 -- Currently accepted permissions are: 'public', 'private' or 'all'
 --
-DROP FUNCTION IF EXISTS cdb_usertables(text);
+DROP FUNCTION IF EXISTS CDB_UserTables(text);
 CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
 RETURNS SETOF name
 AS $$
@@ -15,7 +15,7 @@ FROM pg_class c
 JOIN pg_namespace n ON n.oid = c.relnamespace
 WHERE c.relkind = 'r' 
 AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology')
+AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
 AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')

scripts-available/CDB_XYZ.sql

@@ -5,24 +5,9 @@
 CREATE OR REPLACE FUNCTION CDB_XYZ_Resolution(z INTEGER)
 RETURNS FLOAT8
 AS $$
-DECLARE
-  earth_circumference FLOAT8;
-  tile_size INTEGER;
-  full_resolution FLOAT8;
-BEGIN
-
-  -- Earth equatorial circumference in meters (according to wikipedia)
-  earth_circumference := 40075017;
-
-  -- Size of each tile in pixels (1:1 aspect ratio)
-  tile_size := 256;
-
-  full_resolution := earth_circumference/tile_size;
-
-  RETURN full_resolution / (power(2,z));
-
-END
-$$ LANGUAGE 'plpgsql' IMMUTABLE STRICT;
+  -- circumference divided by 256 is z0 resolution, then divide by 2^z
+  SELECT 40075017.0 / 256 / power(2, z);
+$$ LANGUAGE SQL IMMUTABLE STRICT;
 -- }
 
 -- {

scripts-available/CDB_ZoomFromScale.sql

@@ -1,10 +1,9 @@
 CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
 BEGIN
   CASE
-    WHEN scaleDenominator > 1000000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 1000000000 AND scaleDenominator > 500000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 3;
+    WHEN scaleDenominator > 500000000 THEN RETURN 0;
+    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
+    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
     WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
     WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
     WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;

scripts-enabled/241-CDB_GreatCircle.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_GreatCircle.sql

scripts-enabled/242-CDB_Overviews.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Overviews.sql

scripts-enabled/CDB_Conf.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Conf.sql

scripts-enabled/CDB_Groups.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups.sql

scripts-enabled/CDB_Groups_API.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups_API.sql

scripts-enabled/CDB_Stats.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Stats.sql

sql/test_ddl_triggers.sql

@@ -1,140 +0,0 @@
-\set VERBOSITY terse
-
--- Set user quota to infinite
-SELECT CDB_SetUserQuotaInBytes(0);
-
--- Enable ddl triggers
-SELECT cartodb.cdb_enable_ddl_hooks();
-
-create schema c;
-
-SELECT CDB_SetUserQuotaInBytes('c', 0);
-
-CREATE USER cartodb_postgresql_unpriv_user;
-GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
---SELECT session_user, current_user;
-
-----------------------
--- CREATE TABLE
-----------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select 1 as i INTO c.t3;
-
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i
-from c.t3;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)) as age
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
-
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
--- Table with cartodb_id field, see
--- http://github.com/CartoDB/cartodb-postgresql/issues/32
-select 1 as cartodb_id INTO c.t4;
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator
-from c.t4;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now() - updated_at)) as age
-FROM CDB_TableMetadata WHERE tabname = 'c.t4'::regclass;
-
-----------------------------
--- ALTER TABLE RENAME COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-
-select pg_sleep(.1);
-alter table c.t3 rename column the_geom_webmercator to webmerc;
-
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc
-from c.t3;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
-
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
-alter table c.t3 rename column the_geom_webmercator to webmerc2;
-
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
-
-----------------------------
--- ALTER TABLE DROP COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
-alter table c.t3 drop column the_geom_webmercator;
-
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2
-from c.t3;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
-
-----------------------------
--- ALTER TABLE ADD COLUMN
-----------------------------
-SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
-select pg_sleep(.1);
-alter table c.t3 add column id2 int;
-
-select
- cartodb_id, created_at=updated_at as "c=u",
- NOW() - updated_at < '1 secs' as "u<1s",
- the_geom, the_geom_webmercator,
- i, webmerc, webmerc2, id2
-from c.t3;
-
-RESET SESSION AUTHORIZATION;
-select
- tabname::text,
- round(extract('secs' from now()  - updated_at)*10) as agecs
-FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
-
-----------------------------
--- DROP TABLE
-----------------------------
-
-RESET SESSION AUTHORIZATION;
-drop schema c cascade;
-select count(*) from CDB_TableMetadata;
-
-DROP USER cartodb_postgresql_unpriv_user;
-DROP FUNCTION _CDB_UserQuotaInBytes();

sql/test_setup.sql

@@ -1,5 +1,4 @@
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION cartodb;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)

test/CDB_CartodbfyTableTest.sql

@@ -12,11 +12,12 @@ DECLARE
   tmp INTEGER;
   ogc_geom geometry_columns; -- old the_geom record in geometry_columns
   ogc_merc geometry_columns; -- old the_geom_webmercator record in geometry_columns
+  tabtext TEXT;
 BEGIN
 
   -- Save current constraints on geometry columns, if any
-  ogc_geom = ('','','','',0,0,'GEOMETRY'); 
-  ogc_merc = ogc_geom; 
+  ogc_geom = ('','','','',0,0,'GEOMETRY');
+  ogc_merc = ogc_geom;
   sql := 'SELECT gc.* FROM geometry_columns gc, pg_class c, pg_namespace n '
     || 'WHERE c.oid = ' || tabname::oid || ' AND n.oid = c.relnamespace'
     || ' AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname'
@@ -30,24 +31,17 @@ BEGIN
     END IF;
   END LOOP;
 
+  tabtext := Format('%s.%s','public',tabname);
+  RAISE NOTICE 'CARTODBFYING % !!!!', tabtext;
   PERFORM CDB_CartodbfyTable('public', tabname);
+  tabname := tabtext::regclass;
 
   sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
   EXECUTE sql INTO STRICT id;
-  sql := 'SELECT created_at,updated_at,the_geom_webmercator FROM '
+  sql := 'SELECT the_geom_webmercator FROM '
     || tabname::text || ' WHERE cartodb_id = ' || id;
   EXECUTE sql INTO STRICT rec;
 
-  -- Check created_at and updated_at at creation time
-  lag = rec.created_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'created_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-  lag = rec.updated_at - now();
-  IF lag > '1 second' THEN
-    RAISE EXCEPTION 'updated_at not defaulting to now() after insert [ valued % ago ]', lag;
-  END IF;
-
   -- Check the_geom_webmercator trigger
   IF round(st_x(rec.the_geom_webmercator)) != 111319 THEN
     RAISE EXCEPTION 'the_geom_webmercator X is % (expecting 111319)', round(st_x(rec.the_geom_webmercator));
@@ -72,7 +66,7 @@ BEGIN
            ELSE 3857 END as expsrid,
       CASE WHEN gc.f_geometry_column = 'the_geom' THEN ogc_geom.type
            ELSE ogc_merc.type END as exptype, gc.*
-    FROM geometry_columns gc, pg_class c, pg_namespace n 
+    FROM geometry_columns gc, pg_class c, pg_namespace n
     WHERE c.oid = tabname::oid AND n.oid = c.relnamespace
           AND gc.f_table_schema = n.nspname AND gc.f_table_name = c.relname
           AND gc.f_geometry_column IN ( 'the_geom', 'the_geom_webmercator')
@@ -84,7 +78,7 @@ BEGIN
         rec.f_geometry_column, rec.srid, rec.expsrid;
     END IF;
     -- Check TYPE constraint didn't change
-    IF rec.type != rec.exptype THEN
+    IF (rec.type != 'GEOMETRY') AND (rec.type != 'POINT') THEN
       RAISE EXCEPTION 'type of % in geometry_columns is %, expected %',
         rec.f_geometry_column, rec.type, rec.exptype;
     END IF;
@@ -94,7 +88,7 @@ BEGIN
       RAISE EXCEPTION '% entries found for table % in geometry_columns, expected 2', tmp, tabname;
   END IF;
 
-  -- Check GiST index 
+  -- Check GiST index
   sql := 'SELECT a.attname, count(ri.relname) FROM'
     || ' pg_index i, pg_class c, pg_class ri, pg_attribute a, pg_opclass o'
     || ' WHERE i.indrelid = c.oid AND ri.oid = i.indexrelid'
@@ -115,10 +109,10 @@ BEGIN
   -- Check null constraint on cartodb_id, created_at, updated_at
   SELECT count(*) FROM pg_attribute a, pg_class c WHERE c.oid = tabname::oid
     AND a.attrelid = c.oid AND NOT a.attisdropped AND a.attname in
-      ( 'cartodb_id', 'created_at', 'updated_at' )
+      ( 'cartodb_id' )
     AND NOT a.attnotnull INTO strict tmp;
   IF tmp > 0 THEN
-      RAISE EXCEPTION 'cartodb_id or created_at or updated_at are missing not-null constraint';
+      RAISE EXCEPTION 'cartodb_id is missing not-null constraint';
   END IF;
 
   -- Cleanup
@@ -170,34 +164,48 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- table with existing updated_at and created_at fields ot type text
-CREATE TABLE t AS SELECT NOW()::text as created_at,
-                         NOW()::text as updated_at,
-                         NOW() as reftime;
-SELECT CDB_CartodbfyTableCheck('t', 'text timestamps');
-SELECT extract(secs from reftime-created_at),
-       extract(secs from reftime-updated_at) FROM t;
-CREATE VIEW v AS SELECT * FROM t;
-SELECT CDB_CartodbfyTableCheck('t', 'cartodbfied with view');
-DROP VIEW v;
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text
+-- CREATE TABLE t AS SELECT 10::text as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+-- select cartodb_id/2 FROM t;
+-- DROP TABLE t;
 
--- table with existing cartodb_id field of type text
-CREATE TABLE t AS SELECT 10::text as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
-select cartodb_id/2 FROM t; 
-DROP TABLE t;
-
--- table with existing cartodb_id field of type text not casting
-CREATE TABLE t AS SELECT 'nan' as cartodb_id;
-SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
-select cartodb_id,_cartodb_id0 FROM t; 
-DROP TABLE t;
+-- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
+-- -- table with existing cartodb_id field of type text not casting
+-- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
+-- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+-- select cartodb_id,_cartodb_id0 FROM t;
+-- DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
 SELECT CDB_CartodbfyTableCheck('t', 'unsequenced cartodb_id');
-select cartodb_id FROM t; 
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column
+CREATE TABLE t AS SELECT 'SRID=4326;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, no SRS
+CREATE TABLE t AS SELECT 'POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, no srs');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text geometry column, unusual SRS
+CREATE TABLE t AS SELECT 'SRID=26910;POINT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, srs = 26819');
+SELECT cartodb_id FROM t;
+DROP TABLE t;
+
+-- table with text unparseable geometry column
+CREATE TABLE t AS SELECT 'SRID=26910;PONT(1 1)'::text AS the_geom, 1::int4 as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text the_geom column, unparseable content');
+SELECT cartodb_id FROM t;
 DROP TABLE t;
 
 -- table with existing cartodb_id serial primary key
@@ -208,21 +216,150 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
 AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
 DROP TABLE t;
 
--- table with existing the_geom and created_at and containing null values
--- Really, a test for surviving an longstanding PostgreSQL bug:
--- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
-CREATE TABLE t (
-    the_geom geometry(Geometry,4326),
-    created_at timestamptz,
-    updated_at timestamptz
+-- tables can be renamed and there's no index name clashing #123
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+ALTER TABLE original RENAME TO original_renamed;
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+DROP TABLE original_renamed;
+DROP TABLE original;
+
+-- Table always have a default seq value after cartodbfy #138
+CREATE TABLE bug_empty_table_no_seq (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
 );
-COPY t (the_geom, created_at, updated_at) FROM stdin;
-0106000020E610000001000000010300000001000000050000009EB8244146435BC017B65E062AD343409EB8244146435BC0F51AF6E2708044400B99891683765AC0F51AF6E2708044400B99891683765AC017B65E062AD343409EB8244146435BC017B65E062AD34340	2012-06-06 21:59:08	2013-06-10 20:17:20
-0106000020E61000000100000001030000000100000005000000DA7763431A1A5CC0FBCEE869313C3A40DA7763431A1A5CC09C1B8F55BC494440F9F4A9C7993356C09C1B8F55BC494440F9F4A9C7993356C0FBCEE869313C3A40DA7763431A1A5CC0FBCEE869313C3A40	2012-06-06 21:59:08	2013-06-10 20:17:20
-\N	\N	\N
-\.
-SELECT CDB_CartodbfyTableCheck('t', 'null geom and timestamp values');
-DROP TABLE t;
+SELECT CDB_CartodbfyTableCheck('bug_empty_table_no_seq', 'Table always have a default seq value after cartodbfy #138');
+INSERT INTO bug_empty_table_no_seq DEFAULT VALUES;
+DROP TABLE bug_empty_table_no_seq;
+
+-- Existing cartodb_id values are respected
+CREATE table existing_cartodb_id (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+INSERT INTO existing_cartodb_id (cartodb_id, description) VALUES
+       (10, 'a'),
+       (20, 'b'),
+       (30, 'c');
+SELECT CDB_CartodbfyTableCheck('existing_cartodb_id', 'Existing cartodb_id values are respected #138');
+SELECT cartodb_id,the_geom,the_geom_webmercator,description,name from existing_cartodb_id;
+DROP TABLE existing_cartodb_id;
+
+-- Table with both the_geom and wkb_geometry
+CREATE TABLE many_geometry_columns (
+       the_geom geometry,
+       wkb_geometry geometry(MultiPoint,4326),
+       description varchar
+);
+INSERT INTO many_geometry_columns (the_geom, wkb_geometry) VALUES
+       ('0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440', '0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440'),
+       ('0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440', '0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440');
+SELECT CDB_CartodbfyTableCheck('many_geometry_columns', 'Table with both the_geom and wkb_geometry #141');
+SELECT * FROM many_geometry_columns;
+DROP TABLE many_geometry_columns;
+
+-- Many colliding geom columns
+CREATE TABLE many_colliding_columns (
+  the_geom varchar,
+  the_geom_webmercator varchar,
+  my_geom geometry,
+  my_mercgeom geometry(Point, 3857),
+  cartodb_id varchar,
+  my_pk integer primary key
+);
+INSERT INTO many_colliding_columns VALUES (
+ 'foo', 'bar', 'SRID=4326;POINT(0 0)', 'SRID=3857;POINT(0 0)', 'nerf', 1
+);
+SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141');
+DROP TABLE many_colliding_columns;
+
+-- Table with null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+-- Table with non unique cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+-- Table with non unique and null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_0 from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  cartodb_id integer
+);
+CREATE UNIQUE INDEX "test_cartodb_id_key" ON test (cartodb_id);
+CREATE UNIQUE INDEX "test_cartodb_id_pkey" ON test (cartodb_id);
+ALTER TABLE test ADD CONSTRAINT "test_pkey" PRIMARY KEY USING INDEX test_cartodb_id_pkey;
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with primary key and unique index on it #174');
+SELECT cartodb_id from test;
+DROP TABLE test;
+
+CREATE TABLE test (
+  name varchar,
+  "first.value" integer,
+  "second.value" integer
+);
+INSERT INTO test VALUES ('one', 1, 2), ('two', 3, 4);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with dots in name columns (cartodb #6114)');
+SELECT name, "first.value" from test;
+DROP TABLE test;
+
+SET client_min_messages TO notice;
+-- _CDB_create_cartodb_id_column with cartodb_id integer already present
+CREATE TABLE test (cartodb_id integer);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+
+-- _CDB_create_cartodb_id_column with cartodb_id text already present
+CREATE TABLE test (cartodb_id text);
+
+SELECT _CDB_Create_Cartodb_ID_Column('test'::regclass);
+SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND column_name = '_cartodb_id0';
+
+DROP TABLE test;
+SET client_min_messages TO error;
 
 -- TODO: table with existing custom-triggered the_geom
 

test/CDB_CartodbfyTableTest_expect

@@ -2,8 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 7 at PERFORM
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
@@ -28,30 +28,109 @@ trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
 DROP TABLE
 SELECT 1
-text timestamps cartodbfied fine
-0|0
-CREATE VIEW
-cartodbfied with view cartodbfied fine
-DROP VIEW
-DROP TABLE
-SELECT 1
-text cartodb_id cartodbfied fine
-5
-DROP TABLE
-SELECT 1
-uncasting text cartodb_id cartodbfied fine
-1|nan
-DROP TABLE
-SELECT 1
 unsequenced cartodb_id cartodbfied fine
 1
 DROP TABLE
+SELECT 1
+text the_geom column cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, no srs cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, srs = 26819 cartodbfied fine
+1
+DROP TABLE
+SELECT 1
+text the_geom column, unparseable content cartodbfied fine
+1
+DROP TABLE
 CREATE TABLE
 cartodb_id serial primary key cartodbfied fine
 t_pkey|cartodb_id
 DROP TABLE
 CREATE TABLE
-null geom and timestamp values cartodbfied fine
+original
+ALTER TABLE
+CREATE TABLE
+original
 DROP TABLE
+DROP TABLE
+CREATE TABLE
+Table always have a default seq value after cartodbfy #138 cartodbfied fine
+INSERT 0 1
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Existing cartodb_id values are respected #138 cartodbfied fine
+10|||a|
+20|||b|
+30|||c|
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with both the_geom and wkb_geometry #141 cartodbfied fine
+1|0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440|0104000020110F00000100000001010000004A9F662B456D5FC11392690DC3F75241|
+2|0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440|0104000020110F00000100000001010000002858E0EC376D5FC1CAE8DB4B95F55241|
+DROP TABLE
+CREATE TABLE
+INSERT 0 1
+Many colliding columns #141 cartodbfied fine
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|3
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Table with non unique cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|2
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with non unique and null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|2
+DROP TABLE
+CREATE TABLE
+CREATE INDEX
+CREATE INDEX
+ALTER TABLE
+INSERT 0 3
+Table with primary key and unique index on it #174 cartodbfied fine
+1
+2
+3
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with dots in name columns (cartodb #6114) cartodbfied fine
+one|1
+two|3
+DROP TABLE
+SET
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field does not have an associated sequence, renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+CREATE TABLE
+NOTICE:  Column cartodb_id already exists
+NOTICE:  Existing cartodb_id field is of invalid type text (need int2, int4 or int8), renaming
+NOTICE:  Trying to recover data from _cartodb_id0 column
+
+DROP TABLE
+SET
 DROP FUNCTION
 DROP FUNCTION

test/CDB_DateToNumberTest.sql

@@ -0,0 +1,2 @@
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00'::timestamp);
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+05'::timestamp with time zone);

test/CDB_DateToNumberTest_expect

@@ -0,0 +1,2 @@
+915753600
+915735600

test/CDB_GreatCircle.sql

@@ -0,0 +1,2 @@
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059)));
+select ST_AsText(CDB_GreatCircle(CDB_LatLng(55.8580,4.2590), CDB_LatLng(40.7127,74.0059), 50000));

test/CDB_GreatCircle_expect

@@ -0,0 +1,2 @@
+LINESTRING(4.259 55.858,5.6692453115051 56.0150275120673,7.10720375678704 56.157400475677,8.5718366560563 56.2842986378254,10.0619272412891 56.3949153508462,11.5760785994189 56.4884642014437,13.1127142001617 56.564185934303,14.6700812655504 56.6213555706215,16.2462571744128 56.6592896061102,17.8391590143095 56.6773531596105,19.4465562981665 56.6749669334121,21.0660867567155 56.6516138405427,22.6952750058883 56.6068451534252,24.3315537765309 56.540286032869,25.9722872888145 56.4516403065472,27.6147962622065 56.3406943817481,29.2563839799455 56.207320197769,30.8943627796619 56.0514771479657,32.5260803224591 55.8732129290618,34.1489450028345 55.6726633044968,35.7604499005266 55.4500507979281,37.3581947399686 55.2056823610616,38.9399054089486 54.9399460854786,40.5034506895044 54.6533070499613,42.0468559644411 54.3463024122038,43.5683137754523 54.0195358662507,45.066191217402 53.673671594382,46.5390342525062 53.3094278446298,47.9855691138079 52.9275702630659,49.4047010366934 52.5289051040742,50.7955106088955 52.1142724327331,52.1572480633875 51.6845394219955,53.4893258557794 51.2405938343407,54.7913098701049 50.7833377637432,56.0629095865715 50.3136816997865,57.3039675245588 49.8325389621027,58.5144482465496 49.3408205404538,59.6944271762829 48.8394303639846,60.8440794494795 48.329261012675,61.9636689799149 47.811189874886,63.0535378889196 47.2860757471582,64.1140964137264 46.7547558660385,65.1458133802427 46.2180433565989,66.1492072992903 45.676725078361,67.1248381223566 45.131559846414,68.0732996734468 44.583277003498,68.9952127576034 44.0325753175597,69.8912189338183 43.4801221786776,70.7619749300985 42.9265530691612,71.6081476710291 42.3724712809595,72.4304098829428 41.8184478551838,73.2294362384225 41.2650217194684,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.96060044865294 55.9382939511593,5.6692453115051 56.0150275120673,6.38482117645567 56.0880973218335,7.10720375678705 56.157400475677,7.83625773770865 56.2228347173136,8.5718366560563 56.2842986378254,9.31378281572326 56.3416918804739,10.0619272412891 56.3949153508462,10.8160896721679 56.4438714316548,11.5760785994189 56.4884642014437,12.3416913471456 56.528599656387,13.1127142001617 56.5641859343031,13.8889225793161 56.5951335399513,14.6700812655504 56.6213555706215,15.4559446734179 56.6427679409819,16.2462571744128 56.6592896061102,17.0407534700619 56.6708427815999,17.8391590143095 56.6773531596105,18.6411904842936 56.6787501197174,19.4465562981665 56.6749669334121,20.2549571781681 56.6659409611101,21.0660867567155 56.6516138405428,21.8796322228404 56.6319316654367,22.6952750058883 56.6068451534252,23.5126914929996 56.5763098021872,24.3315537765309 56.5402860328691,25.1515304272452 56.4987393199198,25.9722872888145 56.4516403065472,26.7934882889404 56.3989649050969,27.6147962622065 56.3406943817482,28.4358737796488 56.2768154250305,29.2563839799456 56.207320197769,30.0759913971174 56.1322063721813,30.8943627796619 56.0514771479657,31.7111678961496 55.9651412533344,32.5260803224592 55.8732129290618,33.3387782060384 55.7757118957345,34.1489450028345 55.6726633044969,34.9562701828379 55.5640976716962,35.7604499005266 55.4500507979282,36.5611876268714 55.3305636720814,37.3581947399687 55.2056823610617,38.1511910717861 55.0754578859583,38.9399054089486 54.9399460854787,39.7240759459355 54.7992074675415,40.5034506895044 54.6533070499613,41.277787813601 54.5023141912026,42.0468559644411 54.3463024122039,42.8104345158644 54.1853492102971,43.5683137754524 54.0195358662508,44.3202951422663 53.8489472454711,45.066191217402 53.6736715943821,45.8058258688602 53.4938003329924,46.5390342525062 53.3094278446299,47.2656627911282 53.1206512637978,47.985569113808 52.9275702630661,48.6986219579803 52.7302868398744,49.4047010366934 52.5289051040743,50.1036968736777 52.3235310669909,50.7955106088955 52.1142724327332,51.4800537772815 51.9012383924278,52.1572480633875 51.6845394219956,52.8270250346284 51.4642870840378,53.4893258557795 51.2405938343408,54.1441009873167 51.0135728334539,54.791309870105 50.7833377637434,55.4309205988438 50.5500026522693,56.0629095865715 50.3136816997866,56.6872612224038 50.0744891161178,57.3039675245588 49.8325389621028,57.9130277905821 49.5879449982851,58.5144482465496 49.3408205404539,59.1082416968843 49.091278322122,59.6944271762829 48.8394303639847,60.2730296051101 48.5853878503721,60.8440794494795 48.3292610126751,61.40761238711 48.0711590197009,61.9636689799149 47.8111898748862,62.5122943541616 47.5494603202768,63.0535378889195 47.2860757471584,63.5874529134047 47.0211401132109,64.1140964137264 46.7547558660387,64.6335287494427 46.4870238729191,65.1458133802426 46.2180433565991,65.6510166029904 45.9479118369597,66.1492072992903 45.6767250783612,66.6404566936622 45.4045770424768,67.1248381223566 45.1315598464143,67.6024268127789 44.8577637259253,68.0732996734467 44.5832770034983,68.5375350943572 44.3081860611283,68.9952127576034 44.0325753175599,69.4464134580461 43.7565272097974,69.8912189338183 43.4801221786779,70.3297117064144 43.2034386583077,70.7619749300985 42.9265530691615,71.1880922503468 42.6495398146493,71.6081476710291 42.3724712809597,72.0222254300221 42.0954178399905,72.4304098829428 41.8184478551841,72.8327853946822 41.5416276900883,73.2294362384225 41.2650217194687,73.6204465018146 40.9886923428039,74.0059 40.7127)

test/CDB_HelperTest.sql

@@ -0,0 +1,128 @@
+-- Test unique identifier creation with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', NULL);
+
+-- Test unique identifier creation with prefix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'relname', NULL);
+
+-- Test unique identifier creation with suffix with normal length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'relname', '_suffix');
+
+-- Test unique identifier creation with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test unique identifier creation with prefix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
+DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargolar;
+
+-- Test unique identifier creation with suffix with long length normal relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
+DROP TABLE largolargolargolargolargolargolargolargolargolargolar_suffix;
+
+-- Test unique identifier creation with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', NULL);
+
+-- Test unique identifier creation with prefix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piraña', NULL);
+
+-- Test unique identifier creation with suffix with normal length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', '_suffix');
+
+-- Test unique identifier creation with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test unique identifier creation with prefix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL);
+DROP TABLE prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi;
+
+-- Test unique identifier creation with suffix with long length UTF8 relname
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix (name text);
+SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix');
+DROP TABLE piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'colname', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'colname', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length normal colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargolar text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length normal colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (largolargolargolargolargolargolargolargolargolargolar_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+CREATE TABLE test (name text);
+-- Test unique identifier creation with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piraña', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with suffix with normal length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piraña', '_suffix', 'test'::regclass);
+
+-- Test unique identifier creation with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+
+-- Test unique identifier creation with prefix with long length UTF8 colname
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', NULL, 'test'::regclass);
+DROP TABLE test;
+
+-- Test unique identifier creation with suffix with long length UTF8 colname
+CREATE TABLE test (name text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test new identifier is found when name is taken from previous case
+CREATE TABLE test (piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix text);
+SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácidpin', '_suffix', 'test'::regclass);
+DROP TABLE test;
+
+-- Test _CDB_Octet_Truncate simple case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 5);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 6);
+
+-- Test _CDB_Octet_Truncate UTF8 case
+SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 7);

test/CDB_HelperTest_expect

@@ -0,0 +1,59 @@
+relname
+prefix_relname
+relname_suffix
+largolargolargolargolargolargolargolargolargolargolargolargo
+prefix_largolargolargolargolargolargolargolargolargolargolar
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargolar_0
+DROP TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix_0
+DROP TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+DROP TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+DROP TABLE
+CREATE TABLE
+colname
+prefix_colname
+colname_suffix
+largolargolargolargolargolargolargolargolargolargolargolargo
+prefix_largolargolargolargolargolargolargolargolargolargolar
+DROP TABLE
+CREATE TABLE
+prefix_largolargolargolargolargolargolargolargolargolargolar_0
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix
+DROP TABLE
+CREATE TABLE
+largolargolargolargolargolargolargolargolargolargolar_suffix_0
+DROP TABLE
+CREATE TABLE
+piraña
+prefix_piraña
+piraña_suffix
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
+DROP TABLE
+CREATE TABLE
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
+DROP TABLE
+CREATE TABLE
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+DROP TABLE
+pira
+pirañ
+piraña

test/CDB_OverviewsTest.sql

@@ -0,0 +1,33 @@
+SET client_min_messages TO error;
+\set VERBOSITY default
+
+\i test/overviews/fixtures.sql
+
+SELECT _CDB_Aggregable_Attributes_Expression('base_bare_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
+
+SELECT CDB_CreateOverviews('base_bare_t'::regclass);
+SELECT count(*) FROM _vovw_5_base_bare_t;
+
+
+SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
+SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
+
+SELECT CDB_CreateOverviews('base_t'::regclass);
+SELECT count(*) FROM _vovw_5_base_t;
+
+SELECT CDB_CreateOverviews('polyg_t'::regclass);
+
+SELECT CDB_Overviews('base_t'::regclass);
+SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
+SELECT CDB_Overviews('polyg_t'::regclass);
+
+SELECT CDB_DropOverviews('base_bare_t'::regclass);
+SELECT CDB_DropOverviews('base_t'::regclass);
+SELECT count(*) FROM _vovw_5_base_t;
+
+DROP TABLE base_bare_t;
+DROP TABLE base_t;
+DROP TABLE polyg_t;

test/CDB_OverviewsTest_expect

@@ -0,0 +1,44 @@
+SET
+CREATE TABLE
+INSERT 0 1114
+CREATE TABLE
+INSERT 0 1114
+CREATE TABLE
+INSERT 0 5
+
+
+
+{_vovw_5_base_bare_t,_vovw_4_base_bare_t,_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+125
+number,int_number,name,start
+AVG(number)::double precision AS number,AVG(int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
+AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(tab.name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+125
+
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_t,3,_vovw_3_base_t)
+(base_t,4,_vovw_4_base_t)
+(base_t,5,_vovw_5_base_t)
+(base_bare_t,0,_vovw_0_base_bare_t)
+(base_bare_t,1,_vovw_1_base_bare_t)
+(base_bare_t,2,_vovw_2_base_bare_t)
+(base_bare_t,3,_vovw_3_base_bare_t)
+(base_bare_t,4,_vovw_4_base_bare_t)
+(base_bare_t,5,_vovw_5_base_bare_t)
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_t,3,_vovw_3_base_t)
+(base_t,4,_vovw_4_base_t)
+(base_t,5,_vovw_5_base_t)
+
+
+ERROR:  relation "_vovw_5_base_t" does not exist
+LINE 1: SELECT count(*) FROM _vovw_5_base_t;
+                             ^
+DROP TABLE
+DROP TABLE
+DROP TABLE

test/CDB_QueryTablesTest_expect

@@ -1,4 +1,4 @@
-SELECT * FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}
+SELECT * FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_constraint,pg_catalog.pg_namespace,pg_catalog.pg_type}
 SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)|{pg_catalog.pg_attribute,pg_catalog.pg_class}
 CREATE table "my'tab;le" as select 1|{}
 SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
@@ -20,4 +20,4 @@ select * from sc.test|{sc.test}
 DROP TABLE
 DROP SCHEMA
 SELECT
-* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}
+* FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_constraint,pg_catalog.pg_namespace,pg_catalog.pg_type}

test/CDB_QuotaTest.sql

@@ -1,22 +1,31 @@
 set client_min_messages to error;
 \set VERBOSITY default
 
+-- See the dice
+SELECT setseed(0.5);
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
       EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);
 SELECT CDB_CartodbfyTable('big');
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
-INSERT INTO big SELECT generate_series(1,2048);
+INSERT INTO big SELECT generate_series(2049,4096);
+INSERT INTO big SELECT generate_series(4097,6144);
+INSERT INTO big SELECT generate_series(6145,8192);
+-- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
+SELECT CDB_UserDataSize();
+SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
+-- END Test for #108
+SELECT setseed(0.9);
 SELECT CDB_SetUserQuotaInBytes(2);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);
-INSERT INTO big VALUES (1);
+INSERT INTO big VALUES (8194);
 DROP TABLE big;
 set client_min_messages to NOTICE;
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,15 +1,20 @@
 SET
+
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
 ERROR:  Quota exceeded by 3.9990234375KB
 0
+big
+INSERT 0 2048
+INSERT 0 2048
+INSERT 0 2048
+454656
+909312
+0
 
-INSERT 0 2048
-INSERT 0 2048
-INSERT 0 2048
 2
-ERROR:  Quota exceeded by 567.998046875KB
+ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE

test/CDB_StatsTest.sql

@@ -0,0 +1,13 @@
+-- continuous uniform distribution has kurtosis = -6/5, skewness = 0.0
+-- http://mathworld.wolfram.com/UniformDistribution.html
+set client_min_messages to ERROR;
+
+WITH dist AS (
+  SELECT generate_series(0,10000)::numeric / 10000.0 i
+)
+SELECT
+  abs(CDB_Kurtosis(array_agg(i)) + 1.2) < 1e-3 AS kurtosis,
+  abs(CDB_Skewness(array_agg(i))) < 1e-3 AS skewness
+FROM dist;
+
+set client_min_messages to NOTICE;

test/CDB_StatsTest_expect

@@ -0,0 +1,3 @@
+SET
+t|t
+SET

test/CDB_XYZ_ExtentTest.sql

@@ -4,6 +4,6 @@ range AS ( select z, generate_series(0, pow(2,z)::int-1) as r FROM zoom),
 inp AS ( select z0.z, r1.r as x, r2.r as y FROM zoom z0, range r1, range r2 WHERE z0.z = r1.z and r1.z = r2.z ),
 ext AS ( select x,y,z,CDB_XYZ_Extent(x,y,z) as g from inp )
 select X::text || ',' || Y::text || ',' || Z::text as xyz,
-       st_xmin(g), st_xmax(g), st_ymin(g), st_ymax(g)
+       round(st_xmin(g)), round(st_xmax(g)), round(st_ymin(g)), round(st_ymax(g))
        from ext order by xyz;
 

test/CDB_XYZ_ExtentTest_expect

@@ -1,21 +1,21 @@
-0,0,0|-20037508.5|20037508.5|-20037508.5|20037508.5
-0,0,1|-20037508.5|0|0|20037508.5
-0,0,2|-20037508.5|-10018754.25|10018754.25|20037508.5
-0,1,1|-20037508.5|0|-20037508.5|0
-0,1,2|-20037508.5|-10018754.25|0|10018754.25
-0,2,2|-20037508.5|-10018754.25|-10018754.25|0
-0,3,2|-20037508.5|-10018754.25|-20037508.5|-10018754.25
-1,0,1|0|20037508.5|0|20037508.5
-1,0,2|-10018754.25|0|10018754.25|20037508.5
-1,1,1|0|20037508.5|-20037508.5|0
-1,1,2|-10018754.25|0|0|10018754.25
-1,2,2|-10018754.25|0|-10018754.25|0
-1,3,2|-10018754.25|0|-20037508.5|-10018754.25
-2,0,2|0|10018754.25|10018754.25|20037508.5
-2,1,2|0|10018754.25|0|10018754.25
-2,2,2|0|10018754.25|-10018754.25|0
-2,3,2|0|10018754.25|-20037508.5|-10018754.25
-3,0,2|10018754.25|20037508.5|10018754.25|20037508.5
-3,1,2|10018754.25|20037508.5|0|10018754.25
-3,2,2|10018754.25|20037508.5|-10018754.25|0
-3,3,2|10018754.25|20037508.5|-20037508.5|-10018754.25
+0,0,0|-20037508|20037508|-20037508|20037508
+0,0,1|-20037508|0|0|20037508
+0,0,2|-20037508|-10018754|10018754|20037508
+0,1,1|-20037508|0|-20037508|0
+0,1,2|-20037508|-10018754|0|10018754
+0,2,2|-20037508|-10018754|-10018754|0
+0,3,2|-20037508|-10018754|-20037508|-10018754
+1,0,1|0|20037508|0|20037508
+1,0,2|-10018754|0|10018754|20037508
+1,1,1|0|20037508|-20037508|0
+1,1,2|-10018754|0|0|10018754
+1,2,2|-10018754|0|-10018754|0
+1,3,2|-10018754|0|-20037508|-10018754
+2,0,2|0|10018754|10018754|20037508
+2,1,2|0|10018754|0|10018754
+2,2,2|0|10018754|-10018754|0
+2,3,2|0|10018754|-20037508|-10018754
+3,0,2|10018754|20037508|10018754|20037508
+3,1,2|10018754|20037508|0|10018754
+3,2,2|10018754|20037508|-10018754|0
+3,3,2|10018754|20037508|-20037508|-10018754

test/extension/run_at_cartodb_schema.sql

@@ -1,4 +1,6 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
 \i scripts-available/CDB_TableMetadata.sql
+\i scripts-available/CDB_ColumnNames.sql
+\i scripts-available/CDB_ColumnType.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -123,7 +123,7 @@ function create_role_and_schema() {
 
 function drop_role_and_schema() {
     local ROLE=$1
-    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "DROP SCHEMA \"${ROLE}\" CASCADE;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
     sql "DROP ROLE \"${ROLE}\";"
 }
@@ -178,6 +178,7 @@ function setup() {
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
     sql "CREATE EXTENSION postgis;"
+    sql "CREATE EXTENSION plpythonu;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
@@ -209,8 +210,8 @@ function tear_down() {
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
-    sql 'DROP SCHEMA cdb_testmember_1;'
-    sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_testmember_1 CASCADE;'
+    sql 'DROP SCHEMA cdb_testmember_2 CASCADE;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
@@ -227,7 +228,12 @@ function run_tests() {
     local TESTS
     if [[ $# -ge 1 ]]
     then
-        TESTS="$@"
+        if [[ $# -eq 1 ]]
+        then
+            TESTS=`cat $0 | grep -o "$1[^\(]*"`
+        else
+            TESTS="$@"
+        fi
     else
         TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
     fi
@@ -337,6 +343,92 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_column_names() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
+
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "carto"
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "db"
+
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames'::regclass) c) as s" should "carto"
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_2.table_cnames') c) as s" should "db"
+
+    # Using schema from owner
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" should "carto"
+
+    ## it's not possible to get column names from a table where you don't have permissions
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" fails
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_cnames'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_cnames'
+}
+
+function test_cdb_column_type() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_ctype(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_ctype(c text, a text, r text, t text, o text);'
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "integer"
+    sql cdb_testmember_2 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "text"
+
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_1.table_ctype', 'c')" should "integer"
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_2.table_ctype', 'c')" should "text"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_ctype'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_ctype'
+}
+
+function test_cdb_querytables_schema_and_table_names_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE SCHEMA "foo.bar";'
+    sql postgres 'CREATE TABLE "foo.bar"."c.a.r.t.o.d.b" (a int);'
+    sql postgres 'INSERT INTO "foo.bar"."c.a.r.t.o.d.b" values (1);'
+    sql postgres 'SELECT a FROM "foo.bar"."c.a.r.t.o.d.b";' should 1
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+
+    sql postgres 'DROP TABLE "foo.bar"."c.a.r.t.o.d.b";'
+    sql postgres 'DROP SCHEMA "foo.bar";'
+}
+
+function test_cdb_querytables_table_name_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE "w.a.d.u.s" (a int);';
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+
+    sql postgres 'DROP TABLE "w.a.d.u.s";';
+}
+
+function test_cdb_querytables_happy_cases() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE wadus (a int);';
+    sql postgres 'CREATE TABLE "FOOBAR" (a int);';
+    sql postgres 'CREATE SCHEMA foo;'
+    sql postgres 'CREATE TABLE foo.wadus (a int);';
+
+    ## See how it does NOT quote anything here
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+
+    ## But it quotes when it's needed even if table name has no dots but was created with quotes
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "FOOBAR"$q$);' should '{"public.\"FOOBAR\""}'
+
+    sql postgres 'DROP TABLE wadus;'
+    sql postgres 'DROP TABLE "FOOBAR";'
+    sql postgres 'DROP TABLE foo.wadus;'
+    sql postgres 'DROP SCHEMA foo;'
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@

test/organization/test.sh

@@ -28,6 +28,7 @@ function clear_partial_result() {
 function sql() {
     local ROLE
     local QUERY
+    ERROR_OUTPUT_FILE='/tmp/test_error.log'
     if [[ $# -ge 2 ]]
     then
         ROLE="$1"
@@ -37,15 +38,40 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as ${ROLE}"
-      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}' as ${ROLE}"
+        RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     else
-      log_debug "Executing query '${QUERY}'"
-      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}'"
+        RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     fi
     CODERESULT=$?
+    ERROR_OUTPUT=`cat "${ERROR_OUTPUT_FILE}"`
+    rm ${ERROR_OUTPUT_FILE}
 
-    echo ${RESULT}
+    echo -n "> Code Result: "
+    echo -n ${CODERESULT}
+    echo -n "; Result: "
+    echo -n ${RESULT}
+    echo -n "; Error output: "
+    echo -n ${ERROR_OUTPUT}
+
+    # Some warnings should actually be failures
+    if [[ ${CODERESULT} == "0" ]]
+    then
+        case "${ERROR_OUTPUT}" in
+            WARNING:*no*privileges*were*granted*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES GRANTING WARNING"
+                CODERESULT=1
+            ;;
+            WARNING:*no*privileges*could*be*revoked*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES REVOKING WARNING"
+                CODERESULT=1
+            ;;
+            *) ;;
+        esac
+        echo -n "; Code result after warnings: "
+        echo -n ${CODERESULT}
+    fi
     echo
 
     if [[ ${CODERESULT} -ne 0 ]]
@@ -133,13 +159,18 @@ function create_table() {
 function setup() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
+    sql "CREATE EXTENSION plpythonu;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
-
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups_API.sql
 
     log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_org_admin
+    sql "SELECT cartodb.CDB_Organization_AddAdmin('cdb_org_admin');"
     create_role_and_schema cdb_testmember_1
     create_role_and_schema cdb_testmember_2
     sql "CREATE ROLE publicuser LOGIN;"
@@ -152,8 +183,16 @@ function setup() {
     create_table cdb_testmember_2 bar
     sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3), (4), (5);'
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_a_tmp')"
+    sql "SELECT cartodb.CDB_Group_RenameGroup('group_a_tmp', 'group_a')"
+
+    sql "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_b')"
 }
 
+
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -162,19 +201,29 @@ function tear_down() {
     sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
     sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
 
+    sql "select cartodb.CDB_Group_DropGroup('group_b')"
+
+    sql "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "select cartodb.CDB_Group_DropGroup('group_a')"
+    sql "SELECT cartodb.CDB_Organization_RemoveAdmin('cdb_org_admin');"
+
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
     sql 'DROP SCHEMA cdb_testmember_1;'
     sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_org_admin;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_org_admin;"
 
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
     sql 'DROP ROLE publicuser;'
+    sql 'DROP ROLE cdb_org_admin;'
 
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
@@ -198,6 +247,7 @@ function run_tests() {
         echo "####################################################################"
         clear_partial_result
         setup
+        log_info "############################# TESTS #############################"
         eval ${t}
         if [[ ${PARTIALOK} -ne 0 ]]
         then
@@ -330,44 +380,167 @@ function test_user_can_read_when_it_has_permission_after_organization_permission
 }
 
 function test_cdb_querytables_returns_schema_and_table_name() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
 }
 
 function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_usertables_should_work_with_orgusers() {
-    sql "GRANT USAGE ON SCHEMA cartodb TO publicuser;"
-    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
+    # This test validates the changes proposed in https://github.com/CartoDB/cartodb/pull/5021
+
+    # create tables
     sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
-    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+    sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
     sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
+    
+    sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"
+
+
+    # this is what we need to make public tables available in CDB_UserTables
+    sql postgres "grant publicuser to cdb_testmember_1;"
+    sql postgres "grant publicuser to cdb_testmember_2;"
+
+
+    # this is required to enable select from other schema
+    sql postgres "GRANT USAGE ON SCHEMA cdb_testmember_1 TO publicuser";
+
+
+    # test CDB_UserTables with publicuser
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_UserTables.sql
+
     sql publicuser "SELECT count(*) FROM CDB_UserTables('all')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('public')" should 1
     sql publicuser "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    sql publicuser "SELECT * FROM CDB_UserTables('all')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql publicuser "SELECT * FROM CDB_UserTables('private')" should ""
     # the following tests are for https://github.com/CartoDB/cartodb-postgresql/issues/98
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('all')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('public')" should 1
-    #sql cdb_testmember_2 "SELECT count(*) FROM CDB_UserTables('private')" should 0
+    # cdb_testmember_2 is already owner of `bar` table
+    sql cdb_testmember_2 "select string_agg(t,',') from (select cdb_usertables('all') t order by t) as s" should "bar,test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('public')" should "test_perms_pub"
+    sql cdb_testmember_2 "SELECT * FROM CDB_UserTables('private')" should "bar"
+
+    # test cdb_testmember_2 can select from cdb_testmember_1's public table
+    sql cdb_testmember_2 "SELECT * FROM cdb_testmember_1.test_perms_pub" should 1
 
     sql cdb_testmember_1 "DROP TABLE test_perms_pub"
     sql cdb_testmember_1 "DROP TABLE test_perms_priv"
 }
 
+function test_CDB_Group_Table_GrantRead_should_grant_select_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_CDB_Group_Table_GrantReadWrite_should_grant_insert_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_cant_be_used_by_normal_members() {
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_CreateGroup('group_x_1');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RenameGroup('group_a', 'group_x_2');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_DropGroup('group_a');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_2']);" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1']);" fails
+}
+
+function test_group_permission_functions_cant_be_used_by_normal_members() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_can_be_used_by_org_admin() {
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_CreateGroup('group_x_tmp');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RenameGroup('group_x_tmp', 'group_x');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_AddUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RemoveUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    # TODO: workaround superadmin limitation
+    sql "SELECT cartodb.CDB_Group_DropGroup('group_x');"
+}
+
+function test_org_admin_cant_grant_permissions_on_tables_he_does_not_own() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_valid_group_names() {
+    sql postgres "select cartodb._CDB_Group_GroupRole('group_1$_a');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('GROUP_1$_A');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('_group_1$_a');"
+}
+
+function test_administrator_name_generation() {
+    sql postgres "select cartodb._CDB_Organization_Admin_Role_Name();"
+}
+
+function test_conf() {
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('test_conf', '{ \"a_key\": \"test_val\" }')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should '{ "a_key": "test_val" }'
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_RemoveConf('test_conf')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+}
 
 #################################################### TESTS END HERE ####################################################
 

test/overviews/gen_points.rb

@@ -0,0 +1,43 @@
+# Ruby script to generate test/overviews/fixtures.sql for testing overviews
+# Generated tables:
+# * base_bare_t -- points without attributes (only PK, geometries)
+
+NUM_CLUSTERS = 128
+MAX_PER_CLUSTER = 16
+CLUSTER_RADIUS = 1E-3
+MIN_X = -10.0
+MAX_X =  10.0
+MIN_Y =  30.0
+MAX_Y = 40.0
+ATTRIBUTES = "number double precision, int_number integer, name text, start date"
+
+id = 0
+POINTS = (0...NUM_CLUSTERS).map{
+  x = MIN_X + rand()*(MAX_X - MIN_X)
+  y = MIN_Y + rand()*(MAX_Y - MIN_Y)
+  (0..rand(MAX_PER_CLUSTER)).map{
+    id += 1
+    {
+      id: id,
+      x: (x + rand()*CLUSTER_RADIUS).round(6),
+      y: (y + rand()*CLUSTER_RADIUS).round(6)
+    }
+  }
+}.flatten
+
+values = POINTS.map{ |point|
+  "#{point[:id]}, 'SRID=4326;POINT(#{point[:x]} #{point[:y]})'::geometry, ST_Transform('SRID=4326;POINT(#{point[:x]} #{point[:y]})'::geometry, 3857)"
+}
+
+File.open('fixtures.sql', 'w') do |sql|
+
+  sql.puts "-- bare table with no attribute columns"
+  sql.puts "CREATE TABLE base_bare_t (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry);"
+  sql.puts "INSERT INTO base_bare_t VALUES"
+  sql.puts values.map{|v| "(#{v})"}.join(",\n") + ";"
+
+  sql.puts "-- table with attributes"
+  sql.puts "CREATE TABLE base_t (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry, #{ATTRIBUTES});"
+  sql.puts "INSERT INTO base_t VALUES"
+  sql.puts values.map{|v| "(#{v})"}.join(",\n") + ";"
+end