New version 0.11.2

Fix schema not being specified on pg_get_serial_sequence

.gitignore

@@ -5,4 +5,5 @@ results/
 regression.*
 expected/test
 sql/test
-.idea/*
+.idea/*
+*.swp

CONTRIBUTING.md

@@ -44,7 +44,7 @@ the extension into your test database.
 
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires cheating with PostgreSQL
-so to enforce re-load of the scripts. To help with cheating, "make install"
+as to enforce the scripts to reload. To help with cheating, "make install"
 also installs migration scripts to go from "V" to "V"next and from "V"next
 to "V". Example to upgrade a 0.2.0dev version:
 

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.9.2
+EXTVERSION = 0.11.2
 
 SED = sed
 
@@ -45,6 +45,14 @@ UPGRADABLE = \
   0.9.0 \
   0.9.1 \
   0.9.2 \
+  0.9.3 \
+  0.9.4 \
+  0.10.0 \
+  0.10.1 \
+  0.10.2 \
+  0.11.0 \
+  0.11.1 \
+  0.11.2 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,47 @@
+next (2015-mm-dd)
+-----------------
+
+0.11.2 (2015-10-19)
+-------------------
+* Fix schema not being specified on pg_get_serial_sequence [#170](https://github.com/CartoDB/cartodb-postgresql/pull/170)
+* Log invalidation function call duration in seconds [#163](https://github.com/CartoDB/cartodb-postgresql/pull/163)
+
+0.11.1 (2015-10-06)
+-------------------
+* Added CDB_DateToNumber(timestamp with time zone) [#169](https://github.com/CartoDB/cartodb-postgresql/pull/169)
+* cartodbfy now discards cartodb_id candidates that contain nulls [#148](https://github.com/CartoDB/cartodb-postgresql/issues/148)
+
+0.11.0 (2015-09-dd)
+-------------------
+* Groups API
+
+0.10.2 (2015-09-24)
+-------------------
+* Add back the `DROP FUNCTION IF EXISTS CDB_UserTables(text);` to be able to upgrade from `0.7.3` upward [#160](https://github.com/CartoDB/cartodb-postgresql/issues/160)
+
+0.10.1 (2015-09-16)
+-------------------
+* Get back the `update_updated_at` function (still used by old tables) [#143](https://github.com/CartoDB/cartodb-postgresql/pull/143)
+* Fix for CDB_StatsTest.sql test failing randomly [#144](https://github.com/CartoDB/cartodb-postgresql/issues/144)
+* Fix for table cartodbfy'ed without default seq value [#138](https://github.com/CartoDB/cartodb-postgresql/issues/138)
+* Fix for cartodbfy error column `the_geom` already exists [#141](https://github.com/CartoDB/cartodb-postgresql/issues/141)
+* Fix for columns with geometry cartodbfy'ed without SRID [#154](https://github.com/CartoDB/cartodb-postgresql/issues/154)
+
+0.10.0 (2015-09-07)
+-----------------
+* Quote schema and table names returned by CDB_QueryTables [#134](https://github.com/CartoDB/cartodb-postgresql/pull/134). Use quote_ident to quote schema and table names when necessary.
+* Fixed CDB_ColumnNames [#122](https://github.com/CartoDB/cartodb-postgresql/issues/122) and CDB_ColumnType [#130](https://github.com/CartoDB/cartodb-postgresql/issues/130) should honor regclass, returning columns for just the table in the schema and not in any other one [#131](https://github.com/CartoDB/cartodb-postgresql/pull/131).
+* Add kurtosis and skewness [#124](https://github.com/CartoDB/cartodb-postgresql/pull/124).
+* Removed `DROP FUNCTION IF EXISTS cdb_usertables(text);` [#129](https://github.com/CartoDB/cartodb-postgresql/pull/129). This was needed for upgrading between 0.7.4 to 0.8.0 but is no longer needed.
+
+0.9.4 (2015-08-28)
+------------------
+* Fixed issue with indices when renaming tables [#123](https://github.com/CartoDB/cartodb-postgresql/issues/123)
+
+0.9.3 (2015-08-27)
+------------------
+* Modify sampling of quota trigger [#126](https://github.com/CartoDB/cartodb-postgresql/issues/126)
+
 0.9.2 (2015-08-24)
 ------------------
 * Fix for `the_geom` column present but not SRID (EWKT) and other corner cases [#121](https://github.com/CartoDB/cartodb-postgresql/pull/121)

README.md

@@ -98,3 +98,8 @@ ls `pg_config --sharedir`/extension/cartodb*
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires special steps documented
 in the CONTRIBUTING document, under "Testing changes live".
+
+Limitations
+-----------
+
+- The main schema of an organization user must have one only owner (the user).

expected/test_ddl_triggers.out

@@ -91,6 +91,7 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc;
+NOTICE:  column "the_geom_webmercator" of relation "t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping
@@ -115,6 +116,7 @@ select pg_sleep(.1);
 (1 row)
 
 alter table c.t3 rename column the_geom_webmercator to webmerc2;
+NOTICE:  column "the_geom_webmercator" of relation "t3" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_create" does not exist, skipping
 NOTICE:  event trigger "cdb_on_relation_drop" does not exist, skipping
 NOTICE:  event trigger "cdb_on_alter_column" does not exist, skipping

scripts-available/CDB_CartodbfyTable.sql

@@ -222,7 +222,7 @@ BEGIN
 
   sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
          || reloid::text
-         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
+         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(0.1, ''-1'', '''
          || schema_name::text
          || ''')';
   EXECUTE sql;
@@ -280,6 +280,16 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql VOLATILE;
 
+--- Trigger to update the updated_at column. No longer added by default
+--- but kept here for compatibility with old tables which still have this behavior
+--- and have it added
+CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
+  RETURNS TRIGGER AS $$
+BEGIN
+   NEW.updated_at := now();
+   RETURN NEW;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
 
 -- Auxiliary function
 CREATE OR REPLACE FUNCTION cartodb._CDB_is_raster_table(schema_name TEXT, reloid REGCLASS)
@@ -352,7 +362,7 @@ $$ LANGUAGE PLPGSQL;
 --     As before, this drops all the metadata and geom sync triggers
 --
 -- (2) _CDB_Has_Usable_Primary_ID()
---     Returns TRUE if it can find a unique integer primary key named
+--     Returns TRUE if it can find a unique and not null integer primary key named
 --    'cartodb_id' or can rename an existing key.
 --     Returns FALSE otherwise.
 --
@@ -497,30 +507,6 @@ BEGIN
   
 END;
 $$ LANGUAGE 'plpgsql';
-    
-
--- Return the geometry SRID from the column metadata or
--- the geometry of the very first entry in a given column.
-CREATE OR REPLACE FUNCTION _CDB_Geometry_SRID(reloid REGCLASS, columnname TEXT)
-RETURNS INTEGER
-AS $$
-DECLARE
-  rec RECORD;
-BEGIN
-
-  RAISE DEBUG 'CDB(%): %', '_CDB_Geometry_SRID', 'entered function';
-  
-  EXECUTE Format('SELECT ST_SRID(%I) AS srid FROM %s LIMIT 1', columnname, reloid::text)
-  INTO rec;
-
-  IF FOUND THEN 
-    RETURN rec.srid;
-  END IF;
-
-  RETURN 0;
-  
-END;
-$$ LANGUAGE 'plpgsql';
 
 
 -- Find out if the table already has a usable primary key
@@ -565,7 +551,7 @@ BEGIN
         RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey);
         RETURN true;
 
-      -- Check and see if the column values are unique, 
+      -- Check and see if the column values are unique and not null, 
       -- if they are, we can use this column...
       ELSE
 
@@ -573,13 +559,17 @@ BEGIN
         useable_key := true;
       
         BEGIN
-          sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_unique UNIQUE (%s)', reloid::text, const.pkey, const.pkey);
+          sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey);
           RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql;
           EXECUTE sql;
           EXCEPTION      
           -- Failed unique check...
           WHEN unique_violation THEN
-            RAISE NOTICE 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
+            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
+            useable_key := false;
+          -- Failed not null check...
+          WHEN not_null_violation THEN
+            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey);
             useable_key := false;
           -- Other fatal error
           WHEN others THEN
@@ -588,7 +578,7 @@ BEGIN
   
         -- Clean up test constraint
         IF useable_key THEN
-          PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_unique', reloid::text, const.pkey));
+          PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey));
 
         -- Move non-unique column out of the way
         ELSE
@@ -654,9 +644,72 @@ END;
 $$ LANGUAGE 'plpgsql';
 
 
-DROP FUNCTION IF EXISTS _CDB_Has_Usable_Geom(regclass);
+CREATE OR REPLACE FUNCTION _CDB_Has_Usable_PK_Sequence(reloid REGCLASS)
+RETURNS BOOLEAN
+AS $$
+DECLARE
+  seq TEXT;
+  const RECORD;
+  has_sequence BOOLEAN = false;
+BEGIN
+
+  const := _CDB_Columns();
+
+  SELECT pg_get_serial_sequence(reloid::text, const.pkey)
+  INTO STRICT seq;
+  has_sequence := seq IS NOT NULL;
+
+  RETURN has_sequence;
+END;
+$$ LANGUAGE 'plpgsql';
+
+-- Return a set of columns that can be candidates to be the_geom[webmercator]
+-- with some extra information to analyze them.
+CREATE OR REPLACE FUNCTION _cdb_geom_candidate_columns(reloid REGCLASS)
+RETURNS TABLE (attname name, srid integer, typname name, desired_attname text, desired_srid integer)
+AS $$
+DECLARE
+  const RECORD;
+BEGIN
+
+  const := _CDB_Columns();
+
+  RETURN QUERY
+    SELECT
+    a.attname,
+    CASE WHEN t.typname = 'geometry' THEN postgis_typmod_srid(a.atttypmod) ELSE NULL END AS srid,
+    t.typname,
+    f.desired_attname, f.desired_srid
+    FROM pg_class c
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    JOIN pg_type t ON a.atttypid = t.oid,
+    (VALUES (const.geomcol, 4326), (const.mercgeomcol, 3857) ) as f(desired_attname, desired_srid)
+    WHERE c.oid = reloid
+    AND a.attnum > 0
+    AND NOT a.attisdropped
+    AND postgis_typmod_srid(a.atttypmod) IN (4326, 3857, 0)
+    ORDER BY t.oid ASC;
+END;
+$$ LANGUAGE 'plpgsql';
+
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = '_cdb_has_usable_geom_record') THEN
+    CREATE TYPE _cdb_has_usable_geom_record
+      AS (has_usable_geoms boolean,
+        text_geom_column boolean,
+        text_geom_column_name text,
+        text_geom_column_srid boolean,
+        has_geom boolean,
+        has_geom_name text,
+        has_mercgeom boolean,
+        has_mercgeom_name text);
+    END IF;
+END$$;
+
+DROP FUNCTION IF EXISTS _CDB_Has_Usable_Geom(REGCLASS);
 CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Geom(reloid REGCLASS)
-RETURNS RECORD
+RETURNS _cdb_has_usable_geom_record
 AS $$
 DECLARE
   r1 RECORD;
@@ -688,20 +741,7 @@ BEGIN
 
   -- Do we have a column we can use?
   FOR r1 IN
-    SELECT 
-    a.attname, 
-    CASE WHEN t.typname = 'geometry' THEN postgis_typmod_srid(a.atttypmod) ELSE NULL END AS srid,
-    t.typname,
-    f.desired_attname, f.desired_srid
-    FROM pg_class c 
-    JOIN pg_attribute a ON a.attrelid = c.oid 
-    JOIN pg_type t ON a.atttypid = t.oid,
-    (VALUES (const.geomcol, 4326), (const.mercgeomcol, 3857) ) as f(desired_attname, desired_srid)
-    WHERE c.oid = reloid
-    AND a.attnum > 0
-    AND NOT a.attisdropped
-    AND postgis_typmod_srid(a.atttypmod) IN (4326, 3857, 0)
-    ORDER BY t.oid ASC
+    SELECT * FROM _cdb_geom_candidate_columns(reloid)
   LOOP
   
     RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('checking column ''%s''', r1.attname);
@@ -756,7 +796,7 @@ BEGIN
 
       -- If it's the right SRID, we can use it in place without
       -- transforming it!
-      IF r1.srid = r1.desired_srid OR _CDB_Geometry_SRID(reloid, r1.attname) = r1.desired_srid THEN
+      IF r1.srid = r1.desired_srid THEN
 
         RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('found acceptable ''%s''', r1.attname);
 
@@ -769,7 +809,7 @@ BEGIN
         END IF;
         
       -- If it's an unknown SRID, we need to know that too
-      ELSIF r1.srid = 0 OR _CDB_Geometry_SRID(reloid, r1.attname) = 0 THEN
+      ELSIF r1.srid = 0 THEN
         
         -- Unknown SRID, we'll have to fill it in later
         text_geom_column_srid := true;
@@ -780,23 +820,13 @@ BEGIN
     
   END LOOP;
 
-  -- If geom is the wrong name, just rename it.
-  IF has_geom AND has_geom_name != const.geomcol THEN  
-    sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %s', reloid::text, has_geom_name, const.geomcol);
-    PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-  END IF;
-
-  -- If mercgeom is the wrong name, just rename it.
-  IF has_mercgeom AND has_mercgeom_name != const.mercgeomcol THEN  
-    sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %s', reloid::text, has_mercgeom_name, const.mercgeomcol);
-    PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-  END IF;
-  
   SELECT 
     -- If table is perfect (no transforms required), return TRUE!
     has_geom AND has_mercgeom AS has_usable_geoms,
     -- If the geometry column is hiding in a text field, return enough info to deal w/ it.
-    text_geom_column, text_geom_column_name, text_geom_column_srid
+    text_geom_column, text_geom_column_name, text_geom_column_srid,
+    -- Return enough info to rename geom columns if needed
+    has_geom, has_geom_name, has_mercgeom, has_mercgeom_name
     INTO rv;
 
   RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('returning %s', rv);
@@ -806,6 +836,7 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
+
 -- Create a copy of the table. Assumes that the "Has usable" functions
 -- have already been run, so that if there is a 'cartodb_id' column, it is
 -- a "good" one, and the same for the geometry columns. If all the required
@@ -840,6 +871,7 @@ DECLARE
   geom_srid INTEGER;
   
   has_usable_primary_key BOOLEAN;
+  has_usable_pk_sequence BOOLEAN;
   
 BEGIN
 
@@ -867,35 +899,58 @@ BEGIN
 
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_primary_key %', has_usable_primary_key;
 
+  -- See if the candidate primary key column has a sequence for default
+  -- values. No usable pk implies has_usable_pk_sequence = false.
+  has_usable_pk_sequence := false;
+  IF has_usable_primary_key THEN
+    SELECT _CDB_Has_Usable_PK_Sequence(reloid)
+    INTO STRICT has_usable_pk_sequence;
+  END IF;
+
   -- See if the geometry columns we need are already available
   -- on the table. If they are, we don't need to do any bulk
   -- transformation of the table, we can just ensure proper
   -- indexes are in place and apply a rename
   SELECT *
   FROM _CDB_Has_Usable_Geom(reloid) 
-  AS (has_usable_geoms boolean, 
-      text_geom_column boolean, 
-      text_geom_column_name text, 
-      text_geom_column_srid boolean)
   INTO STRICT gc;
 
+  -- If geom is the wrong name, just rename it.
+  IF gc.has_geom AND gc.has_geom_name != const.geomcol THEN
+    sql := Format('ALTER TABLE %s DROP COLUMN IF EXISTS %I', reloid::text, const.geomcol);
+    PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
+    sql := Format('ALTER TABLE %s RENAME COLUMN %I TO %I', reloid::text, gc.has_geom_name, const.geomcol);
+    PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
+  END IF;
+
+  -- If mercgeom is the wrong name, just rename it.
+  IF gc.has_mercgeom AND gc.has_mercgeom_name != const.mercgeomcol THEN
+    sql := Format('ALTER TABLE %s DROP COLUMN IF EXISTS %I', reloid::text, const.mercgeomcol);
+    PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
+    sql := Format('ALTER TABLE %s RENAME COLUMN %I TO %I', reloid::text, gc.has_mercgeom_name, const.mercgeomcol);
+    PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
+  END IF;
+
+
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_geoms %', gc.has_usable_geoms;
 
   -- We can only avoid a rewrite if both the key and 
   -- geometry are usable
 
-  -- No table re-write is required, BUT a rename is required to 
+  -- No table re-write is required, BUT a rename is required to
   -- a destination schema, so do that now
-  IF has_usable_primary_key AND gc.has_usable_geoms AND destschema != relschema THEN
-  
-    RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table needs to be moved to schema (%)', destschema;
-    PERFORM _CDB_SQL(Format('ALTER TABLE %s SET SCHEMA %I', reloid::text, destschema), '_CDB_Rewrite_Table');
-    RETURN true;
+  IF has_usable_primary_key AND has_usable_pk_sequence AND gc.has_usable_geoms THEN
+    IF  destschema != relschema THEN
 
-  -- Don't move anything, just make sure our destination information is set right
-  ELSIF has_usable_primary_key AND gc.has_usable_geoms AND destschema = relschema THEN
+      RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table needs to be moved to schema (%)', destschema;
+      PERFORM _CDB_SQL(Format('ALTER TABLE %s SET SCHEMA %I', reloid::text, destschema), '_CDB_Rewrite_Table');
+
+    ELSE
+
+      RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table in the perfect place';
+
+    END IF;
 
-    RAISE DEBUG 'CDB(_CDB_Rewrite_Table): perfect table in the perfect place';
     RETURN true;
 
   END IF;
@@ -1029,15 +1084,11 @@ BEGIN
       )
       SELECT ', ST_Transform(' 
             || t.missing_srid_start || t.attname || t.missing_srid_end
-            || ',4326)::Geometry(' 
-            || t.geomtype 
-            || ',4326) AS '
+            || ',4326)::Geometry(GEOMETRY,4326) AS '
             || const.geomcol
             || ', cartodb.CDB_TransformToWebmercator(' 
             || t.missing_srid_start || t.attname || t.missing_srid_end
-            || ')::Geometry('
-            || t.geomtype
-            || ',3857) AS '
+            || ')::Geometry(GEOMETRY,3857) AS '
             || const.mercgeomcol,
             t.attname
       INTO geom_transform_sql, geom_column_source
@@ -1094,27 +1145,24 @@ BEGIN
   -- Set up the primary key sequence
   -- If we copied the primary key from the original data, we need
   -- to set the sequence to the maximum value of that key
-  IF has_usable_primary_key THEN
-
-    EXECUTE Format('SELECT max(%s) FROM %s',
-            const.pkey, copyname)
-       INTO destseqmax;
-
-    IF FOUND AND destseqmax IS NOT NULL THEN
-      PERFORM _CDB_SQL(Format('SELECT setval(''%s'', %s)', destseq, destseqmax), '_CDB_Rewrite_Table');
-    END IF;
+  EXECUTE Format('SELECT max(%s) FROM %s',
+          const.pkey, copyname)
+     INTO destseqmax;
 
+  IF destseqmax IS NOT NULL THEN
+    PERFORM _CDB_SQL(Format('SELECT setval(''%s'', %s)', destseq, destseqmax), '_CDB_Rewrite_Table');
   END IF;
 
-  -- Make the primary key use the sequence as its default value
-  sql := Format('ALTER TABLE %s ALTER COLUMN %s SET DEFAULT nextval(''%s'')', 
+   -- Make the primary key use the sequence as its default value
+  sql := Format('ALTER TABLE %s ALTER COLUMN %s SET DEFAULT nextval(''%s'')',
           copyname, const.pkey, destseq);
   PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
 
-  -- Make the sequence owned by the table, so when the table drops, 
+  -- Make the sequence owned by the table, so when the table drops,
   -- the sequence does too
   sql := Format('ALTER SEQUENCE %s OWNED BY %s.%s', destseq, copyname, const.pkey);
   PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
+
   
   -- We just made a copy, so we can drop the original now
   sql := Format('DROP TABLE %s', reloid::text);
@@ -1234,7 +1282,7 @@ BEGIN
     AND c.oid = reloid
     AND am.amname != 'gist'
   LOOP
-    sql := Format('CREATE INDEX %s_%s_gix ON %s USING GIST (%s)', relname, rec.attname, reloid::text, rec.attname);
+    sql := Format('CREATE INDEX ON %s USING GIST (%s)', reloid::text, rec.attname);
     PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes');
   END LOOP;
     

scripts-available/CDB_ColumnNames.sql

@@ -3,11 +3,12 @@ CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
 
-    SELECT column_name 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || '';
+    SELECT c.column_name
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_ColumnType.sql

@@ -3,12 +3,13 @@ CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
 
-    SELECT data_type 
-      FROM information_schema.columns 
-      WHERE
-        table_name IN (SELECT CDB_UserTables())
-        AND table_name = '' || $1 || ''
-        AND column_name = '' || quote_ident($2) || '';
+    SELECT c.data_type
+      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
+      WHERE table_name = _tn.relname
+        AND table_schema = _sn.nspname
+        AND column_name = $2
+        AND _tn.oid = $1::oid
+        AND _sn.oid = _tn.relnamespace;
          
 $$ LANGUAGE SQL;
 

scripts-available/CDB_Conf.sql

@@ -0,0 +1,48 @@
+----------------------------------
+-- CONF MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by superadmin user.
+-- Functions needing reading configuration should use SECURITY DEFINER.
+----------------------------------
+
+-- This will trigger NOTICE if cartodb.CDB_CONF already exists
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    CREATE TABLE IF NOT EXISTS cartodb.CDB_CONF ( KEY TEXT PRIMARY KEY, VALUE JSON NOT NULL );
+END
+$$;
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.CDB_CONF', '');
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cartodb.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_GetConf(key text)
+    RETURNS JSON AS $$
+DECLARE
+    value JSON;
+BEGIN
+    EXECUTE 'SELECT VALUE FROM cartodb.CDB_CONF WHERE KEY = $1;' INTO value USING key;
+    RETURN value;
+END
+$$ LANGUAGE PLPGSQL STABLE;

scripts-available/CDB_DateToNumber.sql

@@ -13,3 +13,19 @@ RETURN output;
 END;
 $$
 LANGUAGE 'plpgsql' STABLE STRICT;
+
+-- Convert timestamp with time zone to double precision
+--
+CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp with time zone)
+RETURNS double precision AS $$
+DECLARE output double precision;
+BEGIN
+    BEGIN
+        SELECT extract (EPOCH FROM input) INTO output;
+    EXCEPTION WHEN OTHERS THEN
+        RETURN NULL;
+    END;
+RETURN output;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE STRICT;

scripts-available/CDB_Groups.sql

@@ -0,0 +1,252 @@
+----------------------------------
+-- GROUP MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by org admin. See CDB_Organization_AddAdmin.
+----------------------------------
+
+-- Creates a new group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_CreateGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('CREATE ROLE %I NOLOGIN;', group_role);
+    PERFORM cartodb._CDB_Group_CreateGroup_API(group_name, group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Drops group and everything that role owns
+-- TODO: LIMITATION: in order to drop a role all its owned objects must be dropped before.
+-- Right now this is done with DROP OWNED, which can only be done by a superadmin.
+-- Not even the role creator can drop the role and the objects it owns.
+-- All group owned objects by the group are permissions.
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_DropGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('DROP OWNED BY %I', group_role);
+    EXECUTE format('DROP ROLE IF EXISTS %I', group_role);
+    PERFORM cartodb._CDB_Group_DropGroup_API(group_name);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Renames a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RenameGroup(old_group_name text, new_group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    old_group_role TEXT;
+    new_group_role TEXT;
+BEGIN
+    old_group_role = cartodb._CDB_Group_GroupRole(old_group_name);
+    new_group_role = cartodb._CDB_Group_GroupRole(new_group_name);
+    EXECUTE format('ALTER ROLE %I RENAME TO %I', old_group_role, new_group_role);
+    PERFORM cartodb._CDB_Group_RenameGroup_API(old_group_name, new_group_name, new_group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Adds users to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_AddUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      IF(group_role IS NULL OR user_role IS NULL)
+      THEN
+        RAISE EXCEPTION 'Group role (%) and user role (%) must be already existing', group_role, user_role;
+      END IF;
+      EXECUTE format('GRANT %I TO %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_AddUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Removes users from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RemoveUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      EXECUTE format('REVOKE %I FROM %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_RemoveUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+----------------------------------
+-- TABLE MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by table owners.
+----------------------------------
+
+-- Grants table read permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantRead(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantRead(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantRead(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT ON TABLE %I.%I TO %I', username, table_name, group_role );
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'r');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Grants table write permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantReadWrite(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE %I.%I TO %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, true);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'w');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Granting and revoking permissions on sequences
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_TableSequences_Permission(group_name text, username text, table_name text, do_grant bool)
+    RETURNS VOID AS $$
+DECLARE
+    column_name TEXT;
+    sequence_name TEXT;
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    FOR column_name IN EXECUTE 'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = current_database() AND TABLE_SCHEMA = $1 AND TABLE_NAME = $2 AND COLUMN_DEFAULT LIKE ''nextval%''' USING username, table_name
+    LOOP
+        EXECUTE format('SELECT PG_GET_SERIAL_SEQUENCE(''%I.%I'', ''%I'')', username, table_name, column_name) INTO sequence_name;
+        IF sequence_name IS NOT NULL THEN
+          IF do_grant THEN
+            -- Here %s is needed since sequence_name has quotes
+            EXECUTE format('GRANT USAGE, SELECT, UPDATE ON SEQUENCE %s TO %I', sequence_name, group_role);
+          ELSE
+            EXECUTE format('REVOKE ALL ON SEQUENCE %s FROM %I', sequence_name, group_role);
+          END IF;
+        END IF;
+    END LOOP;
+    RETURN;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Revokes all permissions on a table from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_RevokeAll(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_RevokeAll(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAll(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('REVOKE ALL ON TABLE %I.%I FROM %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, false);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name, username, table_name);
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-----------------------
+-- Helper functions
+-----------------------
+-- Given a group name returns a role. group_name must be a valid PostgreSQL idenfifier. See http://www.postgresql.org/docs/9.2/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_GroupRole(group_name text)
+    RETURNS TEXT AS $$
+DECLARE
+    group_role TEXT;
+    prefix TEXT;
+    max_length constant INTEGER := 63;
+BEGIN
+    prefix = format('%s_g_', cartodb._CDB_Group_ShortDatabaseName());
+    group_role := format('%s%s', prefix, group_name);
+    IF LENGTH(group_role) > max_length
+    THEN
+        RAISE EXCEPTION 'Group name must be shorter. It can''t have more than % characters, but it is longer (%): %', max_length - LENGTH(prefix), length(group_name), group_name;
+    END IF;
+    RETURN group_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Returns the first owner of the schema matching username. Organization user schemas must have one only owner.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_User_RoleFromUsername(username text)
+    RETURNS TEXT AS $$
+DECLARE
+    user_role TEXT;
+BEGIN
+    -- This was preferred, but non-superadmins won't get results
+    -- SELECT SCHEMA_OWNER FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = $1 LIMIT 1'
+    SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = username INTO user_role;
+    RETURN user_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Database names are too long, we need a shorter version for composing role names
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_ShortDatabaseName()
+    RETURNS TEXT AS $$
+DECLARE
+    short_database_name TEXT;
+BEGIN
+    SELECT md5(current_database()) INTO short_database_name;
+    RETURN short_database_name;
+END
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_Groups_API.sql

@@ -0,0 +1,195 @@
+----------------------------------
+-- GROUP METADATA API FUNCTIONS
+--
+-- Meant to be used by CDB_Group_* functions to sync data with the editor.
+-- Requires configuration parameter. Example: SELECT cartodb.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
+----------------------------------
+
+-- TODO: delete this development cleanup before final merge
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_AddMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_RemoveMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_AddMember_API(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_RemoveMember_API(group_name text, username text);
+
+-- Sends the create group request
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_CreateGroup_API(group_name text, group_role text)
+    RETURNS VOID AS
+$$
+    import string
+
+    url = '/api/v1/databases/{0}/groups'
+    body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_DropGroup_API(group_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_AddUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantPermission_API(group_name text, username text, table_name text, access text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    body = '{ "access": "%s" }' % access
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name text, username text, table_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE TYPE _CDB_Group_API_Params AS (
+    host text,
+    port int,
+    timeout int,
+    auth text
+);
+
+-- This must be explicitally extracted because "composite types are currently not supported".
+-- See http://www.postgresql.org/docs/9.3/static/plpython-database.html.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Conf()
+    RETURNS _CDB_Group_API_Params AS
+$$
+    conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('groups_api') conf")[0]['conf']
+    if conf is None:
+      return None
+    else:
+      import json
+      params = json.loads(conf)
+      auth = 'Basic %s' % plpy.execute("SELECT cartodb._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
+      return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Auth(username text, password text)
+    RETURNS TEXT AS
+$$
+    import base64
+    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+-- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
+    RETURNS int AS
+$$
+    import httplib
+
+    params = plpy.execute("select c.host, c.port, c.timeout, c.auth from cartodb._CDB_Group_API_Conf() c;")[0]
+    if params['host'] is None:
+      return None
+
+    headers = { 'Authorization': params['auth'], 'Content-Type': 'application/json', 'X-Forwarded-Proto': 'https' }
+
+    retry = 3
+
+    last_err = None
+    while retry > 0:
+      try:
+        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        database_name = plpy.execute("select current_database();")[0]['current_database']
+        client.request(method, url.format(database_name), body, headers)
+        response = client.getresponse()
+        assert response.status in valid_return_codes
+        return response.status
+      except Exception as err:
+        retry -= 1
+        last_err = err
+        plpy.warning('Retrying after: ' + str(err))
+        client = SD['groups_api_client'] = None
+
+    if last_err is not None:
+      plpy.error('Fatal Group API error: ' + str(last_err))
+      raise last_err
+
+    return None
+$$ LANGUAGE 'plpythonu' VOLATILE;
+revoke all on function cartodb._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_Organizations.sql

@@ -1,14 +1,16 @@
 CREATE OR REPLACE
 FUNCTION cartodb.CDB_Organization_Member_Group_Role_Member_Name()
     RETURNS TEXT
-AS 'SELECT ''cdb_org_member''::text || ''_'' || md5(current_database());'
+AS $$
+    SELECT 'cdb_org_member'::text || '_' || md5(current_database());
+$$
 LANGUAGE SQL IMMUTABLE;
 
 DO LANGUAGE 'plpgsql' $$
 DECLARE
     cdb_org_member_role_name TEXT;
 BEGIN
-    cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
+  cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
   IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_member_role_name )
   THEN
     EXECUTE 'CREATE ROLE "' || cdb_org_member_role_name || '" NOLOGIN;';
@@ -25,6 +27,60 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
+-------------------------------------------------------------------------------
+-- Administrator
+-------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Organization_Admin_Role_Name()
+    RETURNS TEXT
+AS $$
+    SELECT current_database() || '_a'::text;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+-- Administrator role creation on extension install
+DO LANGUAGE 'plpgsql' $$
+DECLARE
+    cdb_org_admin_role_name TEXT;
+BEGIN
+    cdb_org_admin_role_name := cartodb._CDB_Organization_Admin_Role_Name();
+    IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_admin_role_name )
+    THEN
+        EXECUTE format('CREATE ROLE %I CREATEROLE NOLOGIN;', cdb_org_admin_role_name);
+    END IF;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_AddAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('GRANT %I TO %I WITH ADMIN OPTION', cdb_admin_role, cdb_user_role);
+    -- CREATEROLE is not inherited, and is needed for user creation
+    EXECUTE format('ALTER ROLE %I CREATEROLE', cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_RemoveAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('ALTER ROLE %I NOCREATEROLE', cdb_user_role);
+    EXECUTE format('REVOKE %I FROM %I', cdb_admin_role, cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
 
 -------------------------------------------------------------------------------
 -- Sharing tables

scripts-available/CDB_QueryTables.sql

@@ -41,11 +41,11 @@ BEGIN
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
           xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
-      SELECT unnest(x) as p, unnest(s) as sc from inp
+      SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
     LOOP
       -- RAISE DEBUG 'tab: %', rec2.p;
       -- RAISE DEBUG 'sc: %', rec2.sc;
-      tables := array_append(tables, (rec2.sc || '.' || rec2.p));
+      tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;

scripts-available/CDB_Stats.sql

@@ -0,0 +1,47 @@
+--
+-- Calculate basic statistics of a given dataset
+--
+-- @param in_array A numeric array of numbers
+--
+-- Returns: statistical quantity chosen
+-- 
+-- References: http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm
+--
+
+-- Calculate kurtosis
+CREATE OR REPLACE FUNCTION CDB_Kurtosis ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    k numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 4)) / ( $2 * power($3, 4)) - 3
+             FROM (SELECT unnest($4) e ) x'
+    INTO k
+    USING a, c, s, in_array;
+
+    RETURN k;
+END;
+$$ language plpgsql IMMUTABLE;
+
+-- Calculate skewness
+CREATE OR REPLACE FUNCTION CDB_Skewness ( in_array NUMERIC[] ) RETURNS NUMERIC as $$
+DECLARE
+    a numeric;
+    c numeric;
+    s numeric;
+    sk numeric;
+BEGIN
+    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+
+    EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 * power($3, 3))
+             FROM (SELECT unnest($4) e ) x'
+    INTO sk
+    USING a, c, s, in_array;
+
+    RETURN sk;
+END;
+$$ language plpgsql IMMUTABLE;

scripts-available/CDB_TableMetadata.sql

@@ -64,6 +64,10 @@ DECLARE
   tabname TEXT;
   rec RECORD;
   found BOOL;
+  function_start timestamptz;
+  function_end timestamptz;
+  function_duration float;
+  log_error_verbosity_value text;
 BEGIN
 
   IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
@@ -72,6 +76,8 @@ BEGIN
     tabname = OLD.tabname;
   END IF;
 
+  function_start := clock_timestamp();
+ 
   -- Notify table data update
   -- This needs a little bit more of research regarding security issues
   -- see https://github.com/CartoDB/cartodb/pull/241
@@ -105,7 +111,14 @@ BEGIN
     EXIT;
   END LOOP;
   IF NOT found THEN RAISE WARNING 'Missing cdb_invalidate_varnish()'; END IF;
-
+  
+  function_end := clock_timestamp();
+  SELECT extract(epoch from (function_end - function_start)) INTO function_duration;
+  SELECT setting INTO log_error_verbosity_value FROM pg_settings WHERE name='log_error_verbosity';
+  SET log_error_verbosity=TERSE;
+  RAISE LOG 'invalidation_duration: %', function_duration::text;
+  PERFORM 'SET log_error_verbosity= ' || log_error_verbosity_value;
+  
   RETURN NULL;
 END;
 $$

scripts-available/CDB_UserTables.sql

@@ -5,7 +5,7 @@
 --
 -- Currently accepted permissions are: 'public', 'private' or 'all'
 --
-DROP FUNCTION IF EXISTS cdb_usertables(text);
+DROP FUNCTION IF EXISTS CDB_UserTables(text);
 CREATE OR REPLACE FUNCTION CDB_UserTables(perm text DEFAULT 'all')
 RETURNS SETOF name
 AS $$
@@ -15,7 +15,7 @@ FROM pg_class c
 JOIN pg_namespace n ON n.oid = c.relnamespace
 WHERE c.relkind = 'r' 
 AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology')
+AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
 AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')

scripts-enabled/CDB_Conf.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Conf.sql

scripts-enabled/CDB_Groups.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups.sql

scripts-enabled/CDB_Groups_API.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups_API.sql

scripts-enabled/CDB_Stats.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Stats.sql

test/CDB_CartodbfyTableTest.sql

@@ -216,6 +216,110 @@ WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
 AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
 DROP TABLE t;
 
+-- tables can be renamed and there's no index name clashing #123
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+ALTER TABLE original RENAME TO original_renamed;
+CREATE TABLE original();
+SELECT CDB_CartodbfyTable('original');
+DROP TABLE original_renamed;
+DROP TABLE original;
+
+-- Table always have a default seq value after cartodbfy #138
+CREATE TABLE bug_empty_table_no_seq (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+SELECT CDB_CartodbfyTableCheck('bug_empty_table_no_seq', 'Table always have a default seq value after cartodbfy #138');
+INSERT INTO bug_empty_table_no_seq DEFAULT VALUES;
+DROP TABLE bug_empty_table_no_seq;
+
+-- Existing cartodb_id values are respected
+CREATE table existing_cartodb_id (
+  cartodb_id integer,
+  the_geom geometry(Geometry,4326),
+  the_geom_webmercator geometry(Geometry,3857),
+  name text,
+  description text
+);
+INSERT INTO existing_cartodb_id (cartodb_id, description) VALUES
+       (10, 'a'),
+       (20, 'b'),
+       (30, 'c');
+SELECT CDB_CartodbfyTableCheck('existing_cartodb_id', 'Existing cartodb_id values are respected #138');
+SELECT cartodb_id,the_geom,the_geom_webmercator,description,name from existing_cartodb_id;
+DROP TABLE existing_cartodb_id;
+
+-- Table with both the_geom and wkb_geometry
+CREATE TABLE many_geometry_columns (
+       the_geom geometry,
+       wkb_geometry geometry(MultiPoint,4326),
+       description varchar
+);
+INSERT INTO many_geometry_columns (the_geom, wkb_geometry) VALUES
+       ('0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440', '0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440'),
+       ('0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440', '0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440');
+SELECT CDB_CartodbfyTableCheck('many_geometry_columns', 'Table with both the_geom and wkb_geometry #141');
+SELECT * FROM many_geometry_columns;
+DROP TABLE many_geometry_columns;
+
+-- Many colliding geom columns
+CREATE TABLE many_colliding_columns (
+  the_geom varchar,
+  the_geom_webmercator varchar,
+  my_geom geometry,
+  my_mercgeom geometry(Point, 3857),
+  cartodb_id varchar,
+  my_pk integer primary key
+);
+INSERT INTO many_colliding_columns VALUES (
+ 'foo', 'bar', 'SRID=4326;POINT(0 0)', 'SRID=3857;POINT(0 0)', 'nerf', 1
+);
+SELECT CDB_CartodbfyTableCheck('many_colliding_columns', 'Many colliding columns #141');
+DROP TABLE many_colliding_columns;
+
+-- Table with null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1), 
+  (2), 
+  (NULL), 
+  (3);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_1 from test;
+DROP TABLE test;
+
+-- Table with non unique cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1), 
+  (2), 
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
+SELECT cartodb_id, cartodb_id_1 from test;
+DROP TABLE test;
+
+-- Table with non unique and null cartodb_id
+CREATE TABLE test (
+  cartodb_id integer
+);
+INSERT INTO test VALUES
+  (1),
+  (2),
+  (NULL),
+  (2);
+SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
+SELECT cartodb_id, cartodb_id_1 from test;
+DROP TABLE test;
+
+
 -- TODO: table with existing custom-triggered the_geom
 
 DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

test/CDB_CartodbfyTableTest_expect

@@ -51,5 +51,56 @@ CREATE TABLE
 cartodb_id serial primary key cartodbfied fine
 t_pkey|cartodb_id
 DROP TABLE
+CREATE TABLE
+original
+ALTER TABLE
+CREATE TABLE
+original
+DROP TABLE
+DROP TABLE
+CREATE TABLE
+Table always have a default seq value after cartodbfy #138 cartodbfied fine
+INSERT 0 1
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Existing cartodb_id values are respected #138 cartodbfied fine
+10|||a|
+20|||b|
+30|||c|
+DROP TABLE
+CREATE TABLE
+INSERT 0 2
+Table with both the_geom and wkb_geometry #141 cartodbfied fine
+1|0104000020E61000000100000001010000007108B023698052C03CEEA53A2E5D4440|0104000020110F00000100000001010000004A9F662B456D5FC11392690DC3F75241|
+2|0104000020E6100000010000000101000000864C9E57618052C0994F0C7F3C5B4440|0104000020110F00000100000001010000002858E0EC376D5FC1CAE8DB4B95F55241|
+DROP TABLE
+CREATE TABLE
+INSERT 0 1
+Many colliding columns #141 cartodbfied fine
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|3
+DROP TABLE
+CREATE TABLE
+INSERT 0 3
+Table with non unique cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|2
+DROP TABLE
+CREATE TABLE
+INSERT 0 4
+Table with non unique and null cartodb_id #148 cartodbfied fine
+1|1
+2|2
+3|
+4|2
+DROP TABLE
 DROP FUNCTION
 DROP FUNCTION

test/CDB_DateToNumberTest.sql

@@ -0,0 +1,2 @@
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00'::timestamp);
+SELECT * FROM CDB_DateToNumber('1999-01-08 00:00:00+05'::timestamp with time zone);

test/CDB_DateToNumberTest_expect

@@ -0,0 +1,2 @@
+915753600
+915735600

test/CDB_QuotaTest.sql

@@ -1,6 +1,9 @@
 set client_min_messages to error;
 \set VERBOSITY default
 
+-- See the dice
+SELECT setseed(0.5);
+
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
@@ -18,6 +21,7 @@ SELECT CDB_UserDataSize();
 SELECT cartodb._CDB_total_relation_size('public', 'big');
 SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
 -- END Test for #108
+SELECT setseed(0.9);
 SELECT CDB_SetUserQuotaInBytes(2);
 INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);

test/CDB_QuotaTest_expect

@@ -1,4 +1,5 @@
 SET
+
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
@@ -11,6 +12,7 @@ INSERT 0 2048
 454656
 909312
 0
+
 2
 ERROR:  Quota exceeded by 443.998046875KB
 0

test/CDB_StatsTest.sql

@@ -0,0 +1,13 @@
+-- continuous uniform distribution has kurtosis = -6/5, skewness = 0.0
+-- http://mathworld.wolfram.com/UniformDistribution.html
+set client_min_messages to ERROR;
+
+WITH dist AS (
+  SELECT generate_series(0,10000)::numeric / 10000.0 i
+)
+SELECT
+  abs(CDB_Kurtosis(array_agg(i)) + 1.2) < 1e-3 AS kurtosis,
+  abs(CDB_Skewness(array_agg(i))) < 1e-3 AS skewness
+FROM dist;
+
+set client_min_messages to NOTICE;

test/CDB_StatsTest_expect

@@ -0,0 +1,3 @@
+SET
+t|t
+SET

test/extension/run_at_cartodb_schema.sql

@@ -1,4 +1,6 @@
 SET SCHEMA 'cartodb';
 \i scripts-available/CDB_Quota.sql
 \i scripts-available/CDB_TableMetadata.sql
+\i scripts-available/CDB_ColumnNames.sql
+\i scripts-available/CDB_ColumnType.sql
 SET SCHEMA 'public';

test/extension/test.sh

@@ -123,7 +123,7 @@ function create_role_and_schema() {
 
 function drop_role_and_schema() {
     local ROLE=$1
-    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "DROP SCHEMA \"${ROLE}\" CASCADE;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
     sql "DROP ROLE \"${ROLE}\";"
 }
@@ -178,6 +178,7 @@ function setup() {
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
     sql "CREATE EXTENSION postgis;"
+    sql "CREATE EXTENSION plpythonu;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
@@ -209,8 +210,8 @@ function tear_down() {
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
-    sql 'DROP SCHEMA cdb_testmember_1;'
-    sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_testmember_1 CASCADE;'
+    sql 'DROP SCHEMA cdb_testmember_2 CASCADE;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
@@ -227,7 +228,12 @@ function run_tests() {
     local TESTS
     if [[ $# -ge 1 ]]
     then
-        TESTS="$@"
+        if [[ $# -eq 1 ]]
+        then
+            TESTS=`cat $0 | grep -o "$1[^\(]*"`
+        else
+            TESTS="$@"
+        fi
     else
         TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
     fi
@@ -337,6 +343,92 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_column_names() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
+
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "carto"
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('table_cnames') c) as s" should "db"
+
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames'::regclass) c) as s" should "carto"
+    sql postgres "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_2.table_cnames') c) as s" should "db"
+
+    # Using schema from owner
+    sql cdb_testmember_1 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" should "carto"
+
+    ## it's not possible to get column names from a table where you don't have permissions
+    sql cdb_testmember_2 "SELECT string_agg(c,'') from (SELECT cartodb.CDB_ColumnNames('cdb_testmember_1.table_cnames') c) as s" fails
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_cnames'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_cnames'
+}
+
+function test_cdb_column_type() {
+    sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_ctype(c int, a int, r int, t int, o int);'
+    sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_ctype(c text, a text, r text, t text, o text);'
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "integer"
+    sql cdb_testmember_2 "SELECT cartodb.CDB_ColumnType('table_ctype', 'c')" should "text"
+
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_1.table_ctype', 'c')" should "integer"
+    sql postgres "SELECT cartodb.CDB_ColumnType('cdb_testmember_2.table_ctype', 'c')" should "text"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.table_ctype'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.table_ctype'
+}
+
+function test_cdb_querytables_schema_and_table_names_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE SCHEMA "foo.bar";'
+    sql postgres 'CREATE TABLE "foo.bar"."c.a.r.t.o.d.b" (a int);'
+    sql postgres 'INSERT INTO "foo.bar"."c.a.r.t.o.d.b" values (1);'
+    sql postgres 'SELECT a FROM "foo.bar"."c.a.r.t.o.d.b";' should 1
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "foo.bar"."c.a.r.t.o.d.b"$q$);' should '{"\"foo.bar\".\"c.a.r.t.o.d.b\""}'
+
+    sql postgres 'DROP TABLE "foo.bar"."c.a.r.t.o.d.b";'
+    sql postgres 'DROP SCHEMA "foo.bar";'
+}
+
+function test_cdb_querytables_table_name_with_dots() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE "w.a.d.u.s" (a int);';
+
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from "w.a.d.u.s"$q$);' should '{"public.\"w.a.d.u.s\""}'
+
+    sql postgres 'DROP TABLE "w.a.d.u.s";';
+}
+
+function test_cdb_querytables_happy_cases() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+
+    sql postgres 'CREATE TABLE wadus (a int);';
+    sql postgres 'CREATE TABLE "FOOBAR" (a int);';
+    sql postgres 'CREATE SCHEMA foo;'
+    sql postgres 'CREATE TABLE foo.wadus (a int);';
+
+    ## See how it does NOT quote anything here
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from wadus$q$);' should '{public.wadus}'
+    sql postgres 'SELECT CDB_QueryTables($q$select * from foo.wadus$q$);' should '{foo.wadus}'
+
+    ## But it quotes when it's needed even if table name has no dots but was created with quotes
+    sql postgres 'SELECT CDB_QueryTablesText($q$select * from "FOOBAR"$q$);' should '{"public.\"FOOBAR\""}'
+
+    sql postgres 'DROP TABLE wadus;'
+    sql postgres 'DROP TABLE "FOOBAR";'
+    sql postgres 'DROP TABLE foo.wadus;'
+    sql postgres 'DROP SCHEMA foo;'
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@

test/organization/test.sh

@@ -28,6 +28,7 @@ function clear_partial_result() {
 function sql() {
     local ROLE
     local QUERY
+    ERROR_OUTPUT_FILE='/tmp/test_error.log'
     if [[ $# -ge 2 ]]
     then
         ROLE="$1"
@@ -37,15 +38,40 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as ${ROLE}"
-      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}' as ${ROLE}"
+        RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     else
-      log_debug "Executing query '${QUERY}'"
-      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+        log_debug "Executing query '${QUERY}'"
+        RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     fi
     CODERESULT=$?
+    ERROR_OUTPUT=`cat "${ERROR_OUTPUT_FILE}"`
+    rm ${ERROR_OUTPUT_FILE}
 
-    echo ${RESULT}
+    echo -n "> Code Result: "
+    echo -n ${CODERESULT}
+    echo -n "; Result: "
+    echo -n ${RESULT}
+    echo -n "; Error output: "
+    echo -n ${ERROR_OUTPUT}
+
+    # Some warnings should actually be failures
+    if [[ ${CODERESULT} == "0" ]]
+    then
+        case "${ERROR_OUTPUT}" in
+            WARNING:*no*privileges*were*granted*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES GRANTING WARNING"
+                CODERESULT=1
+            ;;
+            WARNING:*no*privileges*could*be*revoked*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES REVOKING WARNING"
+                CODERESULT=1
+            ;;
+            *) ;;
+        esac
+        echo -n "; Code result after warnings: "
+        echo -n ${CODERESULT}
+    fi
     echo
 
     if [[ ${CODERESULT} -ne 0 ]]
@@ -133,13 +159,18 @@ function create_table() {
 function setup() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
+    sql "CREATE EXTENSION plpythonu;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
-
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups_API.sql
 
     log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_org_admin
+    sql "SELECT cartodb.CDB_Organization_AddAdmin('cdb_org_admin');"
     create_role_and_schema cdb_testmember_1
     create_role_and_schema cdb_testmember_2
     sql "CREATE ROLE publicuser LOGIN;"
@@ -152,8 +183,16 @@ function setup() {
     create_table cdb_testmember_2 bar
     sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3), (4), (5);'
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_a_tmp')"
+    sql "SELECT cartodb.CDB_Group_RenameGroup('group_a_tmp', 'group_a')"
+
+    sql "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_b')"
 }
 
+
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -162,19 +201,29 @@ function tear_down() {
     sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
     sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
 
+    sql "select cartodb.CDB_Group_DropGroup('group_b')"
+
+    sql "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "select cartodb.CDB_Group_DropGroup('group_a')"
+    sql "SELECT cartodb.CDB_Organization_RemoveAdmin('cdb_org_admin');"
+
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
     sql 'DROP SCHEMA cdb_testmember_1;'
     sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_org_admin;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_org_admin;"
 
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
     sql 'DROP ROLE publicuser;'
+    sql 'DROP ROLE cdb_org_admin;'
 
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
@@ -198,6 +247,7 @@ function run_tests() {
         echo "####################################################################"
         clear_partial_result
         setup
+        log_info "############################# TESTS #############################"
         eval ${t}
         if [[ ${PARTIALOK} -ne 0 ]]
         then
@@ -330,21 +380,18 @@ function test_user_can_read_when_it_has_permission_after_organization_permission
 }
 
 function test_cdb_querytables_returns_schema_and_table_name() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
 }
 
 function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
@@ -393,6 +440,107 @@ function test_cdb_usertables_should_work_with_orgusers() {
     sql cdb_testmember_1 "DROP TABLE test_perms_priv"
 }
 
+function test_CDB_Group_Table_GrantRead_should_grant_select_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_CDB_Group_Table_GrantReadWrite_should_grant_insert_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_cant_be_used_by_normal_members() {
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_CreateGroup('group_x_1');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RenameGroup('group_a', 'group_x_2');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_DropGroup('group_a');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_2']);" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1']);" fails
+}
+
+function test_group_permission_functions_cant_be_used_by_normal_members() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_can_be_used_by_org_admin() {
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_CreateGroup('group_x_tmp');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RenameGroup('group_x_tmp', 'group_x');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_AddUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RemoveUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    # TODO: workaround superadmin limitation
+    sql "SELECT cartodb.CDB_Group_DropGroup('group_x');"
+}
+
+function test_org_admin_cant_grant_permissions_on_tables_he_does_not_own() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_valid_group_names() {
+    sql postgres "select cartodb._CDB_Group_GroupRole('group_1$_a');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('GROUP_1$_A');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('_group_1$_a');"
+}
+
+function test_administrator_name_generation() {
+    sql postgres "select cartodb._CDB_Organization_Admin_Role_Name();"
+}
+
+function test_conf() {
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('test_conf', '{ \"a_key\": \"test_val\" }')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should '{ "a_key": "test_val" }'
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_RemoveConf('test_conf')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+}
 
 #################################################### TESTS END HERE ####################################################