Merge pull request #67 from CartoDB/faster-quota-check

Optimize CDB_UserDataSize (on behalf of @javisantana) #65

.gitignore

@@ -5,3 +5,4 @@ results/
 regression.*
 expected/test
 sql/test
+.idea/*

.travis.yml

@@ -4,7 +4,9 @@ addons:
   postgresql: 9.3
 
 before_install:
+  - sudo apt-get update
   #- sudo apt-get install -q postgresql-9.3-postgis-2.1
+  - sudo apt-get update
   - sudo apt-get install -q postgresql-server-dev-9.3
   - sudo apt-get install -q postgresql-plpython-9.3
   # Install schema_triggers
@@ -19,5 +21,5 @@ before_install:
 script:
   - make
   - sudo make install
-  - PGOPTIONS='-c client_min_messages=NOTICE' make installcheck ||
+  - PGOPTIONS='-c client_min_messages=NOTICE' PGUSER=postgres make installcheck ||
     { cat regression.diffs; false; }

CONTRIBUTING.md

@@ -0,0 +1,60 @@
+The development tracker for cartodb-postgresql is on github:
+http://github.com/cartodb/cartodb-postgresql/
+
+Bug fixes are best reported as pull requests over there.
+Features are best discussed on the mailing list:
+https://groups.google.com/d/forum/cartodb
+
+Adding features to the extension
+--------------------------------
+
+Extension features are coded in scripts found under the
+"scripts-available" directory. A feature can be a single function
+or a group of function with a specific scope.
+
+The "scripts-enabled" directory contains symlinks to the scripts
+in "scripts-available". Any symlink in that directory is automatically
+included in the extension. Numbering can be used to enforce the order
+in which those scripts are loaded.
+
+Scripts would be best coded in a way to be usable both for creation
+and upgrade of the objects. This means using CREATE OR REPLACE for
+the functions, and whatever it takes to check existence of any previous
+version of objects in other cases.
+
+When used as an extension (probably always from version 0.2.0 onwards)
+all the objects will be installed in a "cartodb" schema. Take this into
+account to fully-qualify internal calls to avoid (possibly dangerous)
+name clashes.
+
+Every new feature (as well as bugfixes) should come with a test case,
+see next section.
+
+Writing testcases
+-----------------
+
+Tests reside in the test/ directory.
+You can find information about how to write tests in test/README
+
+Testing changes live
+--------------------
+
+Testing changes made during development requires upgrading
+the extension into your test database. 
+
+During development the cartodb extension version doesn't change with
+every commit, so testing latest change requires cheating with PostgreSQL
+so to enforce re-load of the scripts. To help with cheating, "make install"
+also installs migration scripts to go from "V" to "V"next and from "V"next
+to "V". Example to upgrade a 0.2.0dev version:
+
+```sql
+ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
+ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
+```
+
+Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
+
+```sql
+SELECT cartodb.cdb_extension_reload();
+```

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.2.0
+EXTVERSION = 0.5.2
 
 SED = sed
 
@@ -17,7 +17,21 @@ UPGRADABLE = \
   unpackaged \
   0.1.0 \
   0.1.1 \
-  0.2.0dev \
+  0.2.0 \
+  0.2.1 \
+  0.3.0 \
+  0.3.0dev \
+  0.3.1 \
+  0.3.2 \
+  0.3.3 \
+  0.3.4 \
+  0.3.5 \
+  0.3.6 \
+  0.4.0 \
+  0.4.1 \
+  0.5.0 \
+  0.5.1 \
+  $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
 
@@ -27,7 +41,6 @@ UPGRADES = \
      $(SED) 's/$$/--$(EXTVERSION).sql/' | \
      $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g')
 
-REV=$(shell git describe)
 GITDIR=$(shell test -d .git && echo '.git' || cat .git | $(SED) 's/^gitdir: //')
 
 DATA_built = \
@@ -69,7 +82,7 @@ $(EXTENSION).control: $(EXTENSION).control.in Makefile
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION) $(REV)/' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
@@ -89,7 +102,13 @@ legacy_regress: $(REGRESS_OLD) Makefile
     cat test/$${tn}_expect >> $${exp}; \
   done
 
+test_organization:
+	bash test/organization/test.sh
+
+test_extension_new:
+	bash test/extension/test.sh
+
 legacy_tests: legacy_regress 
 
-installcheck: legacy_tests
+installcheck: legacy_tests test_extension_new test_organization
 

NEWS

@@ -1,35 +0,0 @@
-0.2.0 - 2014-06-09
-------------------
-
-Important changes:
-
- - This release adds dependency on "plpythonu" extension
- - Roles are not created anymore, previously private functions
-   for table information extraction (CDB_UserTables, CDB_TableIndexes,
-   CDB_ColumnNames, CDB_ColumnType) will now be callable by anyone while
-   only returning information about tables over which the calling user
-   has SELECT privilege (#36)
-
-Bug fixes:
-
- - Fix recursive trigger on create table (#32)
- - Ensure cartodb_id uses an associated sequence (#33)
- - Fully qualify call to cdb_disable_ddl_hooks from cdb_enable_ddl_hooks
- - Fully qualify call to CDB_UserDataSize from quota trigger
- - Fully qualify call to CDB_TransformToWebmercator from CDB_CartodbfyTable
- - Fix potential infinite loop in CDB_CartodbfyTable
- - Fix potential infinite loop in CDB_QueryStatements
-
-Enhancements:
-
- - Include revision info in cdb_version() output (#34)
-
-New features:
-
- - Add a cdb_extension_reload() function
-
-
-0.1.0 - 2014-05-23
-------------------
-
-Initial release

NEWS.md

@@ -0,0 +1,100 @@
+0.5.2 (2015-01-29)
+------------------
+* Improvement: make CDB_UserDataSize functions much faster.
+
+0.5.1 (2014-11-21)
+------------------
+* Bugfix: Quota check and some organization permissions functions were not properly escaping table name.
+
+0.5.0 (2014-11-03)
+------------------
+* Support of raster tables for cartodbfication
+* Modified quota functions: vector tables stay the same, raster tables count as full size (as have no
+  the_geom + the_geom_webmercator combo) and raster overviews are not counted
+
+0.4.1 (2014-09-21)
+------------------
+* Bugfix for Cartodbfication: Set primary key of the table if not already present (e.g. tables created from SQL API)
+
+0.4.0 (2014-08-27)
+------------------
+Added CDB_Math_Mode function
+Changes in versioning: no revision is attached so it no longer uses `git describe` for the version.
+
+0.3.6 (2014-08-11)
+------------------
+Dummy release to solve some issues with cdb branch/tag
+
+0.3.5 (2014-08-11)
+------------------
+Inverting priority of CDB_CheckQuota qmax so gies more priority to existing user quota function over parameter value.
+
+0.3.4 (2014-08-01)
+------------------
+Fixes issue with schemas in CDB_QueryTables
+
+0.3.3 (2014-07-30)
+------------------
+* Splitting of CartodbfyTable method in subfunctions to be able to call in fragments and evade timeouts on hot zones
+
+0.3.2 (2014-07-28)
+------------------
+* Make 0.3.0dev version upgradeable
+
+0.3.1 (2014-07-22)
+------------------
+* Dummy version. We start using semantic versioning
+
+0.3.0 (2014-07-15)
+------------------
+* Permission management functions
+* Adapt functions to use schemas
+
+0.2.1 - 2014-06-11
+------------------
+
+Enhancements:
+
+ - Do not force re-cartodbfication on CREATE FROM unpackaged
+ - Drop useless DEFAULT specification in plpgsql variable declarations
+ - List plpythonu requirement first, to get pg_catalog scanned before public
+
+Bug fixes:
+
+ - Do not add unique index on cartodb_id if already a primary key (#38)
+
+0.2.0 - 2014-06-09
+------------------
+
+Important changes:
+
+ - This release adds dependency on "plpythonu" extension
+ - Roles are not created anymore, previously private functions
+   for table information extraction (CDB_UserTables, CDB_TableIndexes,
+   CDB_ColumnNames, CDB_ColumnType) will now be callable by anyone while
+   only returning information about tables over which the calling user
+   has SELECT privilege (#36)
+
+Bug fixes:
+
+ - Fix recursive trigger on create table (#32)
+ - Ensure cartodb_id uses an associated sequence (#33)
+ - Fully qualify call to cdb_disable_ddl_hooks from cdb_enable_ddl_hooks
+ - Fully qualify call to CDB_UserDataSize from quota trigger
+ - Fully qualify call to CDB_TransformToWebmercator from CDB_CartodbfyTable
+ - Fix potential infinite loop in CDB_CartodbfyTable
+ - Fix potential infinite loop in CDB_QueryStatements
+
+Enhancements:
+
+ - Include revision info in cdb_version() output (#34)
+
+New features:
+
+ - Add a cdb_extension_reload() function
+
+
+0.1.0 - 2014-05-23
+------------------
+
+Initial release

README.md

@@ -20,17 +20,24 @@ Dependencies
 Install
 -------
 
- make all install
+```sh
+make all install
+```
 
 Test installation
 -----------------
 
- make installcheck
+```sh
+make installcheck
+```
 
 NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
       installation of schema_triggers: you need to add ``schema_triggers.so``
       to the ``shared_preload_libraries`` setting in postgresql.conf !
 
+NOTE: you need to run the installcheck as a superuser, use PGUSER
+      env variable if needed, like: PGUSER=postgres make installcheck
+
 Enable database
 ---------------
 
@@ -87,19 +94,5 @@ ls `pg_config --sharedir`/extension/cartodb*
 ```
 
 During development the cartodb extension version doesn't change with
-every commit, so testing latest change requires cheating with PostgreSQL
-so to enforce re-load of the scripts. To help with cheating, "make install"
-also installs migration scripts to go from "V" to "V"next and from "V"next
-to "V". Example to upgrade a 0.2.0dev version:
-
-```sql
-ALTER EXTENSION cartodb UPDATE TO '0.2.0devnext';
-ALTER EXTENSION cartodb UPDATE TO '0.2.0dev';
-```
-
-Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
-
-```sql
-SELECT cartodb.cdb_extension_reload();
-```
-
+every commit, so testing latest change requires special steps documented
+in the CONTRIBUTING document, under "Testing changes live".

cartodb.control.in

@@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
 superuser = true
 relocatable = false
 schema = cartodb
-requires = 'schema_triggers, postgis, plpythonu'
+requires = 'plpythonu, schema_triggers, postgis'

expected/test_ddl_triggers.out

@@ -19,6 +19,12 @@ NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 (1 row)
 
 create schema c;
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+ cdb_setuserquotainbytes 
+-------------------------
+                       0
+(1 row)
+
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -247,7 +253,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 RESET SESSION AUTHORIZATION;
 drop schema c cascade;
-NOTICE:  drop cascades to 2 other objects
+NOTICE:  drop cascades to 3 other objects
 select count(*) from CDB_TableMetadata;
  count 
 -------

scripts-available/CDB_CartodbfyTable.sql

@@ -1,54 +1,40 @@
 -- Depends on:
+--   * CDB_ExtensionUtils.sql
 --   * CDB_TransformToWebmercator.sql
 --   * CDB_TableMetadata.sql
 --   * CDB_Quota.sql
 --   * _CDB_UserQuotaInBytes() function, installed by rails
---     (user.rebuild_quota_trigger, called by rake task
---      cartodb:db:update_test_quota_trigger)
+--     (user.rebuild_quota_trigger, called by rake task cartodb:db:update_test_quota_trigger)
 
--- Update the_geom_webmercator
-CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
-RETURNS trigger AS $$
-BEGIN
-  NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
-  RETURN NEW;
-END;
-$$ LANGUAGE plpgsql VOLATILE;
-
-CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
-RETURNS TRIGGER AS $$
-BEGIN
-   NEW.updated_at := now();
-   RETURN NEW;
-END;
-$$ LANGUAGE plpgsql VOLATILE;
-
--- Ensure a table is a "cartodb" table
--- See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table
-CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(reloid REGCLASS)
-RETURNS void 
+-- 1) Required checks before running cartodbfication
+-- Either will pass silenty or raise an exception
+CREATE OR REPLACE FUNCTION _CDB_check_prerequisites(schema_name TEXT, reloid REGCLASS)
+RETURNS void
 AS $$
 DECLARE
   sql TEXT;
-  rec RECORD;
-  rec2 RECORD;
-  tabinfo RECORD;
-  had_column BOOLEAN;
-  i INTEGER;
-  new_name TEXT;
-  quota_in_bytes INT8;
-  exists_geom_cols BOOLEAN[];
 BEGIN
+  IF cartodb.schema_exists(schema_name) = false THEN
+    RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
+  END IF;
 
   -- TODO: Check that user quota is set ?
   BEGIN
-    PERFORM public._CDB_UserQuotaInBytes();
-  EXCEPTION WHEN undefined_function THEN
-    RAISE EXCEPTION 'Please set user quota before cartodbfying tables.';
+    EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name::text) INTO sql;
+    EXCEPTION WHEN undefined_function THEN
+      RAISE EXCEPTION 'Please set user quota before cartodbfying tables.';
   END;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Drop cartodb triggers (might prevent changing columns)
 
+-- 2) Drop cartodb triggers (might prevent changing columns)
+CREATE OR REPLACE FUNCTION _CDB_drop_triggers(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
   -- "track_updates"
   sql := 'DROP TRIGGER IF EXISTS track_updates ON ' || reloid::text;
   EXECUTE sql;
@@ -66,10 +52,23 @@ BEGIN
   EXECUTE sql;
   sql := 'DROP TRIGGER IF EXISTS test_quota_per_row ON ' || reloid::text;
   EXECUTE sql;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Ensure required fields exist
 
-  -- We need a cartodb_id column
+-- 3) Cartodb_id creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_cartodb_id_column(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+  cartodb_id_name TEXT;
+BEGIN
   << cartodb_id_setup >>
   LOOP --{
     had_column := FALSE;
@@ -77,18 +76,17 @@ BEGIN
       sql := 'ALTER TABLE ' || reloid::text || ' ADD cartodb_id SERIAL NOT NULL UNIQUE';
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
+      cartodb_id_name := 'cartodb_id';
       EXIT cartodb_id_setup;
-    EXCEPTION
-    WHEN duplicate_column THEN
-      RAISE NOTICE 'Column cartodb_id already exists';
-      had_column := TRUE;
-    WHEN others THEN
-      RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
-        reloid, SQLERRM, SQLSTATE;
+      EXCEPTION
+      WHEN duplicate_column THEN
+        RAISE NOTICE 'Column cartodb_id already exists';
+        had_column := TRUE;
+      WHEN others THEN
+        RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
     END;
 
     IF had_column THEN
-
       SELECT pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id')
         AS seq INTO rec2;
 
@@ -96,9 +94,9 @@ BEGIN
       SELECT
         pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') as seq,
         t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
-       WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
-         AND a.attname = 'cartodb_id'
+      WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = 'cartodb_id'
       INTO STRICT rec;
+
       -- 20=int2, 21=int4, 23=int8
       IF rec.oid NOT IN (20,21,23) THEN -- {
         RAISE NOTICE 'Existing cartodb_id field is of invalid type % (need int2, int4 or int8), renaming', rec.typname;
@@ -107,28 +105,27 @@ BEGIN
       ELSE -- }{
         sql := 'ALTER TABLE ' || reloid::text || ' ALTER COLUMN cartodb_id SET NOT NULL';
         IF NOT EXISTS ( SELECT c.conname FROM pg_constraint c, pg_attribute a
-                      WHERE c.conkey = ARRAY[a.attnum] AND c.conrelid = reloid
-                        AND a.attrelid = reloid
-                        AND NOT a.attisdropped
-                        AND a.attname = 'cartodb_id'
-                        AND c.contype = 'u' ) -- unique
+        WHERE c.conkey = ARRAY[a.attnum] AND c.conrelid = reloid
+              AND a.attrelid = reloid
+              AND NOT a.attisdropped
+              AND a.attname = 'cartodb_id'
+              AND c.contype IN ( 'u', 'p' ) ) -- unique or pkey
         THEN
           sql := sql || ', ADD unique(cartodb_id)';
         END IF;
         BEGIN
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
+          cartodb_id_name := 'cartodb_id';
           EXIT cartodb_id_setup;
-        EXCEPTION
-        WHEN unique_violation OR not_null_violation THEN
-          RAISE NOTICE '%, renaming', SQLERRM;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
-            reloid, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN unique_violation OR not_null_violation THEN
+            RAISE NOTICE '%, renaming', SQLERRM;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
         END;
       END IF; -- }
 
-
       -- invalid column, need rename and re-create it
       i := 0;
       << rename_column >>
@@ -138,18 +135,17 @@ BEGIN
           sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN cartodb_id TO ' || new_name;
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
-        EXCEPTION
-        WHEN duplicate_column THEN
-          i := i+1;
-          CONTINUE rename_column;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)',
-            reloid, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN duplicate_column THEN
+            i := i+1;
+            CONTINUE rename_column;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
         END;
+        cartodb_id_name := new_name;
         EXIT rename_column;
       END LOOP; --}
       CONTINUE cartodb_id_setup;
-
     END IF;
   END LOOP; -- }
 
@@ -158,12 +154,11 @@ BEGIN
     RAISE NOTICE 'Trying to recover data from % column', new_name;
     BEGIN
       -- Copy existing values to new field
-      -- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-      --       is also known to be faster for tables with many rows
+      -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
       -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
       sql := 'ALTER TABLE ' || reloid::text
-          || ' ALTER cartodb_id TYPE int USING '
-          || new_name || '::int4';
+             || ' ALTER cartodb_id TYPE int USING '
+             || new_name || '::int4';
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
@@ -178,23 +173,51 @@ BEGIN
 
       -- Reset sequence name
       sql := 'ALTER SEQUENCE ' || rec2.seq::text
-          || ' RESTART WITH ' || rec.max + 1;
+             || ' RESTART WITH ' || rec.max + 1;
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
-      -- Drop old column (all went find if we got here)
+      -- Drop old column (all went fine if we got here)
       sql := 'ALTER TABLE ' || reloid::text || ' DROP ' || new_name;
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
-    EXCEPTION
-    WHEN others THEN
-      RAISE NOTICE 'Could not initialize cartodb_id with existing values: % (%)',
+      EXCEPTION
+      WHEN others THEN
+        RAISE NOTICE 'Could not initialize cartodb_id with existing values: % (%)',
         SQLERRM, SQLSTATE;
     END;
   END IF;
 
-  -- We need created_at and updated_at
+  -- Set primary key of the table if not already present (e.g. tables created from SQL API)
+  IF cartodb_id_name IS NULL THEN
+    RAISE EXCEPTION 'Cartodbfying % (Didnt get cartodb_id field name)', reloid;
+  END IF;
+  BEGIN
+    sql := 'ALTER TABLE ' || reloid::text || ' ADD PRIMARY KEY (cartodb_id)';
+    EXECUTE sql;
+    EXCEPTION
+    WHEN others THEN
+      RAISE DEBUG 'Table % Already had PRIMARY KEY', reloid;
+  END;
+
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 4) created_at and updated_at creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_timestamp_columns(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+BEGIN
+
   FOR rec IN SELECT * FROM ( VALUES ('created_at'), ('updated_at') ) t(cname)
   LOOP --{
     new_name := null;
@@ -203,48 +226,43 @@ BEGIN
       had_column := FALSE;
       BEGIN
         sql := 'ALTER TABLE ' || reloid::text || ' ADD ' || rec.cname
-          || ' TIMESTAMPTZ NOT NULL DEFAULT now()';
+               || ' TIMESTAMPTZ NOT NULL DEFAULT now()';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
         EXIT column_setup;
-      EXCEPTION
-      WHEN duplicate_column THEN
-        RAISE NOTICE 'Column % already exists', rec.cname;
-        had_column := TRUE;
-      WHEN others THEN
-        RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-          reloid, rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN duplicate_column THEN
+          RAISE NOTICE 'Column % already exists', rec.cname;
+          had_column := TRUE;
+        WHEN others THEN
+          RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
       END;
 
       IF had_column THEN
-
         -- Check data type is a TIMESTAMP WITH TIMEZONE
         SELECT t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
-         WHERE a.atttypid = t.oid AND a.attrelid = reloid
-           AND NOT a.attisdropped AND a.attname = rec.cname
+        WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
         INTO STRICT rec2;
         IF rec2.oid NOT IN (1184) THEN -- timestamptz {
-          RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.
-cname, rec2.typname;
+          RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.cname, rec2.typname;
         ELSE -- }{
           -- Ensure data type is a TIMESTAMP WITH TIMEZONE
           sql := 'ALTER TABLE ' || reloid::text
-            || ' ALTER ' || rec.cname
-            || ' SET NOT NULL,'
-            || ' ALTER ' || rec.cname
-            || ' SET DEFAULT now()';
+                 || ' ALTER ' || rec.cname
+                 || ' SET NOT NULL,'
+                 || ' ALTER ' || rec.cname
+                 || ' SET DEFAULT now()';
           BEGIN
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
             EXIT column_setup;
-          EXCEPTION
-          WHEN not_null_violation THEN -- failed not-null
-            RAISE NOTICE '%, renaming', SQLERRM;
-          WHEN cannot_coerce THEN -- failed cast
-            RAISE NOTICE '%, renaming', SQLERRM;
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-              reloid, rec.cname, SQLERRM, SQLSTATE;
+            EXCEPTION
+            WHEN not_null_violation THEN -- failed not-null
+              RAISE NOTICE '%, renaming', SQLERRM;
+            WHEN cannot_coerce THEN -- failed cast
+              RAISE NOTICE '%, renaming', SQLERRM;
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
         END IF; -- }
 
@@ -257,18 +275,17 @@ cname, rec2.typname;
             sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN ' || rec.cname || ' TO ' || new_name;
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
-          EXCEPTION
-          WHEN duplicate_column THEN
-            i := i+1;
-            CONTINUE rename_column;
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (renaming %): % (%)',
+            EXCEPTION
+            WHEN duplicate_column THEN
+              i := i+1;
+              CONTINUE rename_column;
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (renaming %): % (%)',
               reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
           EXIT rename_column;
         END LOOP; --}
         CONTINUE column_setup;
-
       END IF;
     END LOOP; -- }
 
@@ -277,12 +294,11 @@ cname, rec2.typname;
       RAISE NOTICE 'Trying to recover data from % coumn', new_name;
       BEGIN
         -- Copy existing values to new field
-        -- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-        --       is also known to be faster for tables with many rows
+        -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
         -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
         sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
-            || ' TYPE TIMESTAMPTZ USING '
-            || new_name || '::timestamptz';
+               || ' TYPE TIMESTAMPTZ USING '
+               || new_name || '::timestamptz';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
 
@@ -291,21 +307,36 @@ cname, rec2.typname;
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
 
-      EXCEPTION
-      WHEN others THEN
-        RAISE NOTICE 'Could not initialize % with existing values: % (%)',
-          rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN others THEN
+          RAISE NOTICE 'Could not initialize % with existing values: % (%)', rec.cname, SQLERRM, SQLSTATE;
       END;
     END IF; -- }
-
   END LOOP; -- }
 
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 5) the_geom and the_geom_webmercator creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_the_geom_columns(reloid REGCLASS)
+  RETURNS BOOLEAN[]
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+  exists_geom_cols BOOLEAN[];
+BEGIN
   -- We need the_geom and the_geom_webmercator
   FOR rec IN SELECT * FROM ( VALUES ('the_geom',4326), ('the_geom_webmercator',3857) ) t(cname,csrid) LOOP --{
     << column_setup >> LOOP --{
       BEGIN
         sql := 'ALTER TABLE ' || reloid::text || ' ADD ' || rec.cname
-          || ' GEOMETRY(geometry,' || rec.csrid || ')';
+               || ' GEOMETRY(geometry,' || rec.csrid || ')';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
         sql := 'CREATE INDEX ON ' || reloid::text || ' USING GIST ( ' || rec.cname || ')';
@@ -313,25 +344,22 @@ cname, rec2.typname;
         EXECUTE sql;
         exists_geom_cols := array_append(exists_geom_cols, false);
         EXIT column_setup;
-      EXCEPTION
-      WHEN duplicate_column THEN
-        exists_geom_cols := array_append(exists_geom_cols, true);
-        RAISE NOTICE 'Column % already exists', rec.cname;
-      WHEN others THEN
-        RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-          reloid, rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN duplicate_column THEN
+          exists_geom_cols := array_append(exists_geom_cols, true);
+          RAISE NOTICE 'Column % already exists', rec.cname;
+        WHEN others THEN
+          RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
       END;
 
       << column_fixup >>
       LOOP --{
-
         -- Check data type is a GEOMETRY
         SELECT t.typname, t.oid, a.attnotnull,
-               postgis_typmod_srid(a.atttypmod) as srid,
-               postgis_typmod_type(a.atttypmod) as gtype
-         FROM pg_type t, pg_attribute a
-         WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
-           AND a.attname = rec.cname
+          postgis_typmod_srid(a.atttypmod) as srid,
+          postgis_typmod_type(a.atttypmod) as gtype
+        FROM pg_type t, pg_attribute a
+        WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
         INTO STRICT rec2;
 
         IF rec2.typname NOT IN ('geometry') THEN -- {
@@ -340,46 +368,44 @@ cname, rec2.typname;
         END IF; -- }
 
         IF rec2.srid != rec.csrid THEN -- {
-            BEGIN
-              sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
-                || ' TYPE geometry(' || rec2.gtype || ',' || rec.csrid || ') USING ST_Transform('
-                || rec.cname || ',' || rec.csrid || ')';
-              RAISE DEBUG 'Running %', sql;
-              EXECUTE sql;
+          BEGIN
+            sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
+                   || ' TYPE geometry(' || rec2.gtype || ',' || rec.csrid || ') USING ST_Transform('
+                   || rec.cname || ',' || rec.csrid || ')';
+            RAISE DEBUG 'Running %', sql;
+            EXECUTE sql;
             EXCEPTION
             WHEN others THEN
               RAISE NOTICE 'Could not enforce SRID % to column %: %, renaming', rec.csrid, rec.cname, SQLERRM;
               EXIT column_fixup; -- cannot fix, will rename
-            END;
+          END;
         END IF; -- }
 
         -- add gist indices if not there already
         IF NOT EXISTS ( SELECT ir.relname
                         FROM pg_am am, pg_class ir,
-                             pg_class c, pg_index i,
-                             pg_attribute a
+                          pg_class c, pg_index i,
+                          pg_attribute a
                         WHERE c.oid  = reloid AND i.indrelid = c.oid
-                          AND a.attname = rec.cname
-                          AND i.indexrelid = ir.oid AND i.indnatts = 1
-                          AND i.indkey[0] = a.attnum AND a.attrelid = c.oid
-                          AND NOT a.attisdropped AND am.oid = ir.relam
-                          AND am.amname = 'gist' )
+                              AND a.attname = rec.cname
+                              AND i.indexrelid = ir.oid AND i.indnatts = 1
+                              AND i.indkey[0] = a.attnum AND a.attrelid = c.oid
+                              AND NOT a.attisdropped AND am.oid = ir.relam
+                              AND am.amname = 'gist' )
         THEN -- {
           BEGIN
             sql := 'CREATE INDEX ON ' || reloid::text || ' USING GIST ( ' || rec.cname || ')';
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
-          EXCEPTION
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (% index): % (%)',
-              reloid, rec.cname, SQLERRM, SQLSTATE;
+            EXCEPTION
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (% index): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
         END IF; -- }
 
         -- if we reached this line, all went good
         EXIT column_setup;
-
-      END LOOP; -- } column_fixup 
+      END LOOP; -- } column_fixup
 
       -- invalid column, need rename and re-create it
       i := 0;
@@ -390,73 +416,236 @@ cname, rec2.typname;
           sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN ' || rec.cname || ' TO ' || new_name;
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
-        EXCEPTION
-        WHEN duplicate_column THEN
-          i := i+1;
-          CONTINUE rename_column;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)',
-            reloid, rec.cname, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN duplicate_column THEN
+            i := i+1;
+            CONTINUE rename_column;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
         END;
         EXIT rename_column;
       END LOOP; --}
       CONTINUE column_setup;
-
-    END LOOP; -- } column_setup 
-
+    END LOOP; -- } column_setup
   END LOOP; -- } on expected geometry columns
 
-  -- Initialize the_geom with values from the_geom_webmercator
-  -- do this only if the_geom_webmercator was found (not created)
-  -- _and_ the_geom was NOT found.
-  IF exists_geom_cols[2] AND NOT exists_geom_cols[1] THEN
+  RETURN exists_geom_cols;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 6) Initialize the_geom with values from the_geom_webmercator
+-- do this only if the_geom_webmercator was found (not created) and the_geom was NOT found.
+CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_from_the_geom_webmercator(reloid REGCLASS, geom_columns_exist BOOLEAN[])
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+  IF geom_columns_exist[2] AND NOT geom_columns_exist[1] THEN
     sql := 'UPDATE ' || reloid::text || ' SET the_geom = ST_Transform(the_geom_webmercator, 4326) ';
     EXECUTE sql;
   END IF;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Initialize the_geom_webmercator with values from the_geom
-  -- do this only if the_geom was found (not created)
-  -- _and_ the_geom_webmercator was NOT found.
-  IF exists_geom_cols[1] AND NOT exists_geom_cols[2] THEN
+
+-- 7) Initialize the_geom_webmercator with values from the_geom
+-- do this only if the_geom was found (not created) and the_geom_webmercator was NOT found.
+CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_webmercator_from_the_geom(reloid REGCLASS, geom_columns_exist BOOLEAN[])
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+  IF geom_columns_exist[1] AND NOT geom_columns_exist[2] THEN
     sql := 'UPDATE ' || reloid::text || ' SET the_geom_webmercator = public.CDB_TransformToWebmercator(the_geom) ';
     EXECUTE sql;
   END IF;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Re-create all triggers
 
-  -- NOTE: drop/create has the side-effect of re-enabling disabled triggers
-
-  -- "track_updates"
+-- 8.a) Create all triggers
+-- NOTE: drop/create has the side-effect of re-enabling disabled triggers
+CREATE OR REPLACE FUNCTION _CDB_create_triggers(schema_name TEXT, reloid REGCLASS)
+RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+-- "track_updates"
   sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
-      || reloid::text
-      || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
+         || reloid::text
+         || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
   EXECUTE sql;
 
-  -- "update_the_geom_webmercator"
-  -- TODO: why _before_ and not after ?
+-- "update_the_geom_webmercator"
+-- TODO: why _before_ and not after ?
   sql := 'CREATE trigger update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()';
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()';
   EXECUTE sql;
 
-  -- "update_updated_at"
-  -- TODO: why _before_ and not after ?
+-- "update_updated_at"
+-- TODO: why _before_ and not after ?
   sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
   EXECUTE sql;
 
-  -- "test_quota" and "test_quota_per_row"
+-- "test_quota" and "test_quota_per_row"
 
   sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
-      || reloid::text
-      || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1)';
+         || reloid::text
+         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
+         || schema_name::text
+         || ''')';
   EXECUTE sql;
 
   sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001)';
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', '''
+         || schema_name::text
+         || ''')';
   EXECUTE sql;
- 
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- 8.b) Create all raster triggers
+-- NOTE: drop/create has the side-effect of re-enabling disabled triggers
+CREATE OR REPLACE FUNCTION _CDB_create_raster_triggers(schema_name TEXT, reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+-- "track_updates"
+  sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
+         || reloid::text
+         || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
+  EXECUTE sql;
+
+-- "update_updated_at"
+-- TODO: why _before_ and not after ?
+  sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
+  EXECUTE sql;
+
+-- "test_quota" and "test_quota_per_row"
+
+  sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+
+  sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+
+-- Update the_geom_webmercator
+CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
+  RETURNS trigger
+AS $$
+BEGIN
+  NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
+  RETURNS TRIGGER AS $$
+BEGIN
+   NEW.updated_at := now();
+   RETURN NEW;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+
+-- Auxiliary function
+CREATE OR REPLACE FUNCTION cartodb._CDB_is_raster_table(schema_name TEXT, reloid REGCLASS)
+  RETURNS BOOLEAN
+AS $$
+DECLARE
+  sql TEXT;
+  is_raster BOOLEAN;
+  rel_name TEXT;
+BEGIN
+  IF cartodb.schema_exists(schema_name) = FALSE THEN
+    RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
+  END IF;
+
+  SELECT relname FROM pg_class WHERE oid=reloid INTO rel_name;
+
+  BEGIN
+    sql := 'SELECT the_raster_webmercator FROM '
+          || quote_ident(schema_name::TEXT)
+          || '.'
+          || quote_ident(rel_name::TEXT)
+          || ' LIMIT 1';
+    is_raster = TRUE;
+    EXECUTE sql;
+
+    EXCEPTION WHEN undefined_column THEN
+      is_raster = FALSE;
+  END;
+
+  RETURN is_raster;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+
+-- ////////////////////////////////////////////////////
+
+-- Ensure a table is a "cartodb" table (See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table)
+-- Rails code replicates this call at User.cartodbfy()
+CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(schema_name TEXT, reloid REGCLASS)
+RETURNS void 
+AS $$
+DECLARE
+  exists_geom_cols BOOLEAN[];
+  is_raster BOOLEAN;
+BEGIN
+
+  PERFORM cartodb._CDB_check_prerequisites(schema_name, reloid);
+
+  PERFORM cartodb._CDB_drop_triggers(reloid);
+
+  -- Ensure required fields exist
+  PERFORM cartodb._CDB_create_cartodb_id_column(reloid);
+  PERFORM cartodb._CDB_create_timestamp_columns(reloid);
+
+  SELECT cartodb._CDB_is_raster_table(schema_name, reloid) INTO is_raster;
+  IF is_raster THEN
+    PERFORM cartodb._CDB_create_raster_triggers(schema_name, reloid);
+  ELSE
+    SELECT cartodb._CDB_create_the_geom_columns(reloid) INTO exists_geom_cols;
+
+    -- Both only populate if proceeds
+    PERFORM cartodb._CDB_populate_the_geom_from_the_geom_webmercator(reloid, exists_geom_cols);
+    PERFORM cartodb._CDB_populate_the_geom_webmercator_from_the_geom(reloid, exists_geom_cols);
+
+    PERFORM cartodb._CDB_create_triggers(schema_name, reloid);
+  END IF;
+
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(reloid REGCLASS)
+RETURNS void
+AS $$
+BEGIN
+  PERFORM cartodb.CDB_CartodbfyTable('public', reloid);
 END;
 $$ LANGUAGE PLPGSQL;

scripts-available/CDB_DDLTriggers.sql

@@ -4,14 +4,17 @@ CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
 RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_relation_create_eventinfo();
 
   -- We're only interested in real relations
   IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
 
-  RAISE DEBUG 'Relation % of kind % created in namespace oid %',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relnamespace;
+  SELECT nspname FROM pg_namespace WHERE oid=(event_info.new).relnamespace INTO rel_namespace;
+
+  RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
+	 event_info.relation, (event_info.new).relkind, (event_info.new).relname::TEXT, rel_namespace, (event_info.new).relnamespace;
 
   -- We don't want to react to alters triggered by superuser,
   IF current_setting('is_superuser') = 'on' THEN
@@ -22,7 +25,7 @@ BEGIN
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
   -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
@@ -63,6 +66,7 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_column_alter_eventinfo();
 
@@ -80,9 +84,11 @@ BEGIN
     RETURN;
   END IF;
 
+  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
+
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
@@ -100,6 +106,7 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_column_drop_eventinfo();
 
@@ -117,9 +124,11 @@ BEGIN
     RETURN;
   END IF;
 
+  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
+
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 

scripts-available/CDB_DateToNumber.sql

@@ -2,7 +2,7 @@
 --
 CREATE OR REPLACE FUNCTION CDB_DateToNumber(input timestamp)
 RETURNS double precision AS $$
-DECLARE output double precision DEFAULT NULL;
+DECLARE output double precision;
 BEGIN
     BEGIN
         SELECT extract (EPOCH FROM input) INTO output;

scripts-available/CDB_ExtensionUtils.sql

@@ -11,3 +11,10 @@ BEGIN
   EXECUTE sql;
 END;
 $$ language 'plpgsql' VOLATILE;
+
+CREATE OR REPLACE FUNCTION cartodb.schema_exists(schema_name text)
+RETURNS boolean AS
+$$
+  SELECT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = schema_name::text);
+$$
+language sql VOLATILE;

scripts-available/CDB_Math.sql

@@ -0,0 +1,26 @@
+-- CartoDB Math SQL functions
+
+
+-- Mode
+-- https://wiki.postgresql.org/wiki/Aggregate_Mode
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Math_final_mode(anyarray)
+  RETURNS anyelement AS
+$BODY$
+    SELECT a
+    FROM unnest($1) a
+    GROUP BY 1 
+    ORDER BY COUNT(1) DESC, 1
+    LIMIT 1;
+$BODY$
+LANGUAGE 'sql' IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS cartodb.CDB_Math_Mode(anyelement);
+
+CREATE AGGREGATE cartodb.CDB_Math_Mode(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=_CDB_Math_final_mode,
+  INITCOND='{}'
+);
+

scripts-available/CDB_Organizations.sql

@@ -0,0 +1,90 @@
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Member_Group_Role_Member_Name()
+    RETURNS TEXT
+AS 'SELECT ''cdb_org_member''::text || ''_'' || md5(current_database());'
+LANGUAGE SQL IMMUTABLE;
+
+DO LANGUAGE 'plpgsql' $$
+DECLARE
+    cdb_org_member_role_name TEXT;
+BEGIN
+    cdb_org_member_role_name := cartodb.CDB_Organization_Member_Group_Role_Member_Name();
+  IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_member_role_name )
+  THEN
+    EXECUTE 'CREATE ROLE "' || cdb_org_member_role_name || '" NOLOGIN;';
+  END IF;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Create_Member(role_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'GRANT "' || cartodb.CDB_Organization_Member_Group_Role_Member_Name() || '" TO "' || role_name || '"';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+
+-------------------------------------------------------------------------------
+-- Sharing tables
+-------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Add_Table_Read_Permission(from_schema text, table_name text, to_role_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Add_Table_Organization_Read_Permission(from_schema text, table_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'SELECT cartodb.CDB_Organization_Add_Table_Read_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || cartodb.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Add_Table_Read_Write_Permission(from_schema text, table_name text, to_role_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Add_Table_Organization_Read_Write_Permission(from_schema text, table_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'SELECT cartodb.CDB_Organization_Add_Table_Read_Write_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || cartodb.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Remove_Access_Permission(from_schema text, table_name text, to_role_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'REVOKE ALL PRIVILEGES ON TABLE "' || from_schema || '"."' || table_name || '" FROM "' || to_role_name || '"';
+    -- EXECUTE 'REVOKE USAGE ON SCHEMA ' || from_schema || ' FROM "' || to_role_name || '"';
+    -- We need to revoke usage on schema only if we are revoking privileges from the last table where to_role_name has
+    -- any permission granted within the schema from_schema
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_Remove_Organization_Access_Permission(from_schema text, table_name text)
+    RETURNS void
+AS $$
+BEGIN
+    EXECUTE 'SELECT cartodb.CDB_Organization_Remove_Access_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || cartodb.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
+END
+$$ LANGUAGE PLPGSQL VOLATILE;

scripts-available/CDB_QueryTables.sql

@@ -22,7 +22,7 @@ BEGIN
     END IF;
 
     BEGIN
-      EXECUTE 'EXPLAIN (FORMAT XML) ' || rec.q INTO STRICT exp;
+      EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
     EXCEPTION WHEN others THEN
       -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
       --       the affected table ?
@@ -33,14 +33,19 @@ BEGIN
 
     -- Now need to extract all values of <Relation-Name>
 
-    --RAISE DEBUG 'Explain: %', exp;
+    -- RAISE DEBUG 'Explain: %', exp;
 
     FOR rec2 IN WITH
-      inp AS ( SELECT xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x )
-      SELECT unnest(x)::name as p from inp
+      inp AS (
+        SELECT
+          xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
+          xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
+      )
+      SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
     LOOP
-      --RAISE DEBUG 'tab: %', rec2.p;
-      tables := array_append(tables, rec2.p);
+      -- RAISE DEBUG 'tab: %', rec2.p;
+      -- RAISE DEBUG 'sc: %', rec2.sc;
+      tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name);
     END LOOP;
 
     -- RAISE DEBUG 'Tables: %', tables;

scripts-available/CDB_Quota.sql

@@ -1,61 +1,106 @@
 -- Return the estimated size of user data. Used for quota checking.
+CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
+RETURNS bigint AS
+$$
+DECLARE
+  total_size INT8;
+BEGIN
+  WITH raster_tables AS (
+    SELECT o_table_name, r_table_name FROM raster_overviews
+      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
+  ),
+  user_tables AS (
+    SELECT table_name FROM information_schema.tables
+      WHERE table_catalog = current_database() AND table_schema = schema_name
+        AND table_name != 'spatial_ref_sys'
+        AND table_name != 'cdb_tablemetadata'
+        AND table_type = 'BASE TABLE'
+  ),
+  table_cat AS (
+    SELECT
+      table_name,
+      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
+    FROM user_tables
+  ),
+  sizes AS (
+    SELECT COALESCE(INT8(SUM(pg_total_relation_size('"' || schema_name || '"."' || table_name || '"')))) table_size,
+      CASE
+        WHEN is_overview THEN 0
+	WHEN is_raster THEN 1
+	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
+      END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
+  )
+  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
+
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;
+
+
+-- Return the estimated size of user data. Used for quota checking.
+-- Implicit schema version for backwards compatibility
 CREATE OR REPLACE FUNCTION CDB_UserDataSize()
 RETURNS bigint AS
 $$
-  -- TODO: double check this query. Maybe use CDB_TableMetadata for lookup ?
-  --       also, it's "table_name" sounds sensible to search_path
-  --
-  -- NOTE: division by 2 is an hack for the_geom_webmercator
-  --
-  SELECT coalesce(int8(sum(pg_total_relation_size(quote_ident(table_name))) / 2), 0)
-    AS quota
-  FROM information_schema.tables
-  WHERE table_catalog = current_database() AND table_schema = 'public'
-    AND table_name != 'spatial_ref_sys'
-    AND table_name != 'cdb_tablemetadata'
-    AND table_type = 'BASE TABLE';
+  SELECT public.CDB_UserDataSize('public');
 $$
 LANGUAGE 'sql' VOLATILE;
 
+-- Triggers cannot have declared arguments: pbfact float8, qmax int8, schema_name text
 CREATE OR REPLACE FUNCTION CDB_CheckQuota()
 RETURNS trigger AS
 $$
 DECLARE
-
   pbfact float8;
   qmax int8;
+  schema_name text;
   dice float8;
   quota float8;
 BEGIN
+  IF TG_NARGS = 3 THEN
+    schema_name := TG_ARGV[2];
+    IF cartodb.schema_exists(schema_name) = false THEN
+      RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
+    END IF;
+  ELSE
+    schema_name := 'public';
+  END IF;
+
+  -- By default try to use quota function, and if not present then rely on the one specified by params
+  BEGIN
+    EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax;
+  EXCEPTION WHEN undefined_function THEN
+    BEGIN
+      IF TG_NARGS >= 2 AND TG_ARGV[1] <> '-1' THEN
+        qmax := TG_ARGV[1];
+      ELSE
+        RAISE EXCEPTION 'Missing "%"._CDB_UserQuotaInBytes()', schema_name;
+      END IF;
+    END;
+  END;
 
   pbfact := TG_ARGV[0];
-  dice := random();
 
-  -- RAISE DEBUG 'CDB_CheckQuota enter: pbfact=% dice=%', pbfact, dice;
+  dice := random();
 
   IF dice < pbfact THEN
     RAISE DEBUG 'Checking quota on table % (dice:%, needed:<%)', TG_RELID::text, dice, pbfact;
-    BEGIN
-      qmax := public._CDB_UserQuotaInBytes();
-    EXCEPTION WHEN undefined_function THEN
-      IF TG_NARGS > 1 THEN
-        RAISE NOTICE 'Using quota specified via trigger parameter';
-        qmax := TG_ARGV[1];
-      ELSE
-        RAISE EXCEPTION 'Missing _CDB_UserQuotaInBytes(), and no quota provided as parameter';
-      END IF;
-    END;
 
     IF qmax = 0 THEN
       RETURN NEW;
     END IF;
 
-    SELECT public.CDB_UserDataSize() INTO quota;
+    SELECT public.CDB_UserDataSize(schema_name) INTO quota;
     IF quota > qmax THEN
-        RAISE EXCEPTION 'Quota exceeded by %KB', (quota-qmax)/1024;
+      RAISE EXCEPTION 'Quota exceeded by %KB', (quota-qmax)/1024;
     ELSE RAISE DEBUG 'User quota in bytes: % < % (max allowed)', quota, qmax;
     END IF;
-  -- ELSE RAISE DEBUG 'Not checking quota on table % (dice:%, needed:<%)', TG_RELID::text, dice, pbfact;
   END IF;
 
   RETURN NEW;
@@ -63,26 +108,33 @@ END;
 $$
 LANGUAGE 'plpgsql' VOLATILE;
 
-CREATE OR REPLACE FUNCTION CDB_SetUserQuotaInBytes(bytes int8)
+
+CREATE OR REPLACE FUNCTION CDB_SetUserQuotaInBytes(schema_name text, bytes int8)
 RETURNS int8 AS
 $$
 DECLARE
-  current_quota int8;
   sql text;
 BEGIN
-  BEGIN
-    current_quota := public._CDB_UserQuotaInBytes();
-  EXCEPTION WHEN undefined_function THEN
-    current_quota := 0;
-  END;
+  IF cartodb.schema_exists(schema_name::text) = false THEN
+    RAISE EXCEPTION 'Invalid schema name "%"', schema_name::text;
+  END IF;
 
-  sql := 'CREATE OR REPLACE FUNCTION public._CDB_UserQuotaInBytes() '
+  sql := 'CREATE OR REPLACE FUNCTION "' || schema_name::text || '"._CDB_UserQuotaInBytes() '
     || 'RETURNS int8 AS $X$ SELECT ' || bytes
     || '::int8 $X$ LANGUAGE sql IMMUTABLE';
   EXECUTE sql;
 
-  return current_quota;
-
+  return bytes;
 END
 $$
 LANGUAGE 'plpgsql' VOLATILE STRICT;
+
+
+CREATE OR REPLACE FUNCTION CDB_SetUserQuotaInBytes(bytes int8)
+RETURNS int8 AS
+$$
+BEGIN
+  return public.CDB_SetUserQuotaInBytes('public', bytes);
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_StringToDate.sql

@@ -2,7 +2,7 @@
 --
 CREATE OR REPLACE FUNCTION CDB_StringToDate(input character varying)
 RETURNS date AS $$
-DECLARE output DATE DEFAULT NULL;
+DECLARE output DATE;
 BEGIN
     BEGIN
         output := input::date;

scripts-enabled/210-CDB_Organizations.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Organizations.sql

scripts-enabled/220-CDB_Math.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Math.sql

sql/test_ddl_triggers.sql

@@ -8,6 +8,8 @@ SELECT cartodb.cdb_enable_ddl_hooks();
 
 create schema c;
 
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';

test/CDB_CartodbfyTableTest.sql

@@ -30,7 +30,7 @@ BEGIN
     END IF;
   END LOOP;
 
-  PERFORM CDB_CartodbfyTable(tabname); 
+  PERFORM CDB_CartodbfyTable('public', tabname);
 
   sql := 'INSERT INTO ' || tabname::text || '(the_geom) values ( CDB_LatLng(2,1) ) RETURNING cartodb_id';
   EXECUTE sql INTO STRICT id;
@@ -132,7 +132,7 @@ LANGUAGE 'plpgsql';
 
 -- table with single non-geometrical column
 CREATE TABLE t AS SELECT 1::int as a;
-SELECT CDB_CartodbfyTable('t'); -- should fail
+SELECT CDB_CartodbfyTable('public', 't'); -- should fail
 SELECT CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
 SELECT CDB_CartodbfyTableCheck('t', 'single non-geometrical column');
 DROP TABLE t;
@@ -200,6 +200,14 @@ SELECT CDB_CartodbfyTableCheck('t', 'unsequenced cartodb_id');
 select cartodb_id FROM t; 
 DROP TABLE t;
 
+-- table with existing cartodb_id serial primary key
+CREATE TABLE t ( cartodb_id serial primary key );
+SELECT CDB_CartodbfyTableCheck('t', 'cartodb_id serial primary key');
+SELECT c.conname, a.attname FROM pg_constraint c, pg_attribute a
+WHERE c.conrelid = 't'::regclass and a.attrelid = c.conrelid
+AND c.conkey[1] = a.attnum AND NOT a.attisdropped;
+DROP TABLE t;
+
 -- table with existing the_geom and created_at and containing null values
 -- Really, a test for surviving an longstanding PostgreSQL bug:
 -- http://www.postgresql.org/message-id/20140530143150.GA11051@localhost

test/CDB_CartodbfyTableTest_expect

@@ -2,6 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 7 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
@@ -45,6 +47,10 @@ unsequenced cartodb_id cartodbfied fine
 1
 DROP TABLE
 CREATE TABLE
+cartodb_id serial primary key cartodbfied fine
+t_pkey|cartodb_id
+DROP TABLE
+CREATE TABLE
 null geom and timestamp values cartodbfied fine
 DROP TABLE
 DROP FUNCTION

test/CDB_MathTest.sql

@@ -0,0 +1,4 @@
+
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1]) a;

test/CDB_MathTest_expect

@@ -0,0 +1,3 @@
+2
+1
+1

test/CDB_QueryTablesTest.sql

@@ -25,3 +25,9 @@ WITH inp AS ( select 'create table test (a int); insert into test values (1); se
 
 WITH inp AS ( select 'WITH a AS (select * from pg_class) select * from a'::text as q )
  SELECT q, CDB_QueryTables(q) from inp;
+
+CREATE SCHEMA sc;
+create table sc.test (a int);
+insert into sc.test values (1);
+WITH inp AS ( select 'select * from sc.test'::text as q )
+ SELECT q, CDB_QueryTables(q) from inp;

test/CDB_QueryTablesTest_expect

@@ -1,11 +1,15 @@
-SELECT * FROM geometry_columns|{pg_attribute,pg_class,pg_namespace,pg_type}
-SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)|{pg_attribute,pg_class}
+SELECT * FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_namespace,pg_catalog.pg_type}
+SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)|{pg_catalog.pg_attribute,pg_catalog.pg_class}
 CREATE table "my'tab;le" as select 1|{}
-SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_class}
+SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
 SELECT 1 as col1; select 2 as col2|{}
 WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
 ERROR:  relation "nonexistant" does not exist
-begin; select * from pg_class; commit;|{pg_class}
+begin; select * from pg_class; commit;|{pg_catalog.pg_class}
 WARNING:  CDB_QueryTables cannot explain query: select * from test (42P01: relation "test" does not exist)
 ERROR:  relation "test" does not exist
-WITH a AS (select * from pg_class) select * from a|{pg_class}
+WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
+CREATE SCHEMA
+CREATE TABLE
+INSERT 0 1
+select * from sc.test|{sc.test}

test/CDB_QuotaTest.sql

@@ -4,7 +4,7 @@ CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE CDB_CheckQuota(1,1);
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
 INSERT INTO big VALUES (1); -- disallowed, quota exceeds before
 SELECT CDB_SetUserQuotaInBytes(0);

test/CDB_QuotaTest_expect

@@ -6,9 +6,9 @@ ERROR:  Quota exceeded by 3.9990234375KB
 0
 
 INSERT 0 1024
-0
-ERROR:  Quota exceeded by 103.9921875KB
 8
+ERROR:  Quota exceeded by 123.9921875KB
+0
 INSERT 0 1
 DROP TABLE
 SET

test/extension/run_at_cartodb_schema.sql

@@ -0,0 +1,3 @@
+SET SCHEMA 'cartodb';
+\i scripts-available/CDB_Quota.sql
+SET SCHEMA 'public';

test/extension/test.sh

@@ -0,0 +1,284 @@
+#!/bin/sh
+
+#
+# Tests for the extension since version 0.5.0. They don't replace SQL based ones, for now need to run both
+#
+
+# It is expected that you run this script as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_extension
+CMD='echo psql'
+CMD=psql
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+
+function sql() {
+    local ROLE
+    local QUERY
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+    else
+      log_debug "Executing query '${QUERY}'"
+      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+    fi
+    CODERESULT=$?
+
+    echo ${RESULT}
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}');"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+
+function create_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local RASTER_COL="the_raster_webmercator"
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${OVERVIEW_TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "SELECT AddOverviewConstraints('${ROLE}','${OVERVIEW_TABLENAME}','${RASTER_COL}','${ROLE}','${TABLENAME}','${RASTER_COL}',2);"
+}
+
+function drop_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "drop_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+
+    sql ${ROLE} "DROP TABLE ${ROLE}.${OVERVIEW_TABLENAME};"
+    sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
+}
+
+
+function setup() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    sql "CREATE SCHEMA cartodb;"
+    sql "GRANT USAGE ON SCHEMA cartodb TO public;"
+    sql "CREATE EXTENSION postgis;"
+
+    log_info "########################### BOOTSTRAP ###########################"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
+    ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
+
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+
+    create_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5), (6);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo;'
+
+    create_table cdb_testmember_2 bar
+    sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3);'
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+}
+
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1;'
+    sql 'DROP SCHEMA cdb_testmember_2;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        TESTS="$@"
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+        clear_partial_result
+        setup
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+        tear_down
+    done
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+#################################################### TESTS GO HERE ####################################################
+
+
+# Tests quota checking taking into account both geom and raster tables
+function test_quota_for_each_user() {
+    # Normal tables add 4096 bytes
+    # Raster tables with overview constraints add 16384 bytes
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+
+    create_raster_table cdb_testmember_1 raster_1
+    create_raster_table cdb_testmember_2 raster_2
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 20480
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    create_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 36864
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    drop_raster_table cdb_testmember_1 raster_1
+    drop_raster_table cdb_testmember_2 raster_2
+    drop_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+}
+
+#################################################### TESTS END HERE ####################################################
+
+run_tests $@
+
+exit ${OK}

test/organization/test.sh

@@ -0,0 +1,355 @@
+#!/bin/sh
+
+#
+# It is expected that you run this script
+# as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_organizations
+CMD='echo psql'
+CMD=psql
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+
+function sql() {
+    local ROLE
+    local QUERY
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+    else
+      log_debug "Executing query '${QUERY}'"
+      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+    fi
+    CODERESULT=$?
+
+    echo ${RESULT}
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}')"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+
+function setup() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    sql "CREATE SCHEMA cartodb;"
+    sql "GRANT USAGE ON SCHEMA cartodb TO public;"
+
+    log_info "########################### BOOTSTRAP ###########################"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+
+    create_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo;'
+
+    create_table cdb_testmember_2 bar
+    sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3), (4), (5);'
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+}
+
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1;'
+    sql 'DROP SCHEMA cdb_testmember_2;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        TESTS="$@"
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+        clear_partial_result
+        setup
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+        tear_down
+    done
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+
+#################################################### TESTS GO HERE ####################################################
+
+function test_member_2_cannot_read_without_permission() {
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+}
+
+function test_member_1_cannot_grant_read_permission_to_other_schema_than_its_one() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_2', 'foo', 'cdb_testmember_2')" fails
+}
+
+function test_member_1_grants_read_permission_and_member_2_can_read() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.bar;' fails
+}
+
+function test_member_2_cannot_add_table_to_member_1_schema_after_table_permission_added() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 "CREATE TABLE cdb_testmember_1.bar ( a int );" fails
+}
+
+function test_grant_read_permission_between_two_members() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.bar;' should 5
+}
+
+function test_member_2_cannot_write_to_member_1_table() {
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails
+}
+
+function test_member_1_cannot_grant_read_write_permission_to_other_schema_than_its_one() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_2', 'foo', 'cdb_testmember_2')" fails
+}
+
+function test_member_2_can_write_to_member_1_table_after_write_permission_is_added() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+}
+
+function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_1.foo;' fails
+}
+
+function test_member_1_removes_access_and_member_2_can_no_longer_write_to_the_table() {
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);' fails
+}
+
+function test_giving_permissions_to_two_tables_and_removing_from_first_table_should_not_remove_from_second() {
+    #### test setup
+    # create an extra table for cdb_testmember_1
+    create_table cdb_testmember_1 foo_2
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo_2 VALUES (1), (2), (3), (4), (5);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo_2;'
+
+    # gives read permission to both tables
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo_2', 'cdb_testmember_2')"
+
+    # cdb_testmember_2 has access to both tables
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo_2;' should 5
+
+    # cdb_testmember_1 removes access to foo table
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+
+    # cdb_testmember_2 should have access to foo_2 table but not to table foo
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo_2;' should 5
+
+
+    #### test tear down
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo_2;'
+}
+
+function test_cdb_org_member_role_allows_reading_to_all_users_without_explicit_permission() {
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Add_Table_Organization_Read_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+}
+
+function test_user_can_read_when_it_has_permission_after_organization_permission_is_removed() {
+    create_role_and_schema cdb_testmember_3
+
+    # shares with cdb_testmember_2 and can read but cdb_testmember_3 cannot
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+
+    # granting to organization allows to read to both: cdb_testmember_2 and cdb_testmember_3
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Add_Table_Organization_Read_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+
+    # removing access from organization should keep permission on cdb_testmember_2 but drop it to cdb_testmember_3
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Remove_Organization_Access_Permission('cdb_testmember_1', 'foo');"
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
+    sql cdb_testmember_3 'SELECT count(*) FROM cdb_testmember_1.foo;' fails
+
+    drop_role_and_schema cdb_testmember_3
+}
+
+function test_cdb_querytables_returns_schema_and_table_name() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
+}
+
+function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
+function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
+#################################################### TESTS END HERE ####################################################
+
+
+
+run_tests $@
+
+exit ${OK}

util/create_from_unpackaged.sh

@@ -4,7 +4,22 @@ ver=$1
 input=cartodb--${ver}.sql
 output=cartodb--unpackaged--${ver}.sql
 
-cat ${input} | grep -v 'duplicated extension$' > ${output}
+echo "-- Script generated by $0 on `date`" > ${output}
+
+# Migrate CDB functions from public schema to cartodb schema
+cat ${input} |
+  grep '^ *CREATE OR REPLACE FUNCTION' |
+  grep -v ' cartodb\.' | # should  only match DDL hooks
+  sed 's/).*$/)/' |
+  sed 's/DEFAULT [^ ,)]*//g' |
+  sed 's/CREATE OR REPLACE FUNCTION /ALTER FUNCTION public./' |
+  sed 's/$/ SET SCHEMA cartodb;/' |
+  sed 's/^/DO LANGUAGE plpgsql \$\$ BEGIN /' |
+  sed "s/$/ EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Got % (%)', SQLERRM, SQLSTATE; END; \$\$;/" |
+  cat >> ${output}
+
+# Upgrade all functions
+cat ${input} | grep -v 'duplicated extension$' >> ${output}
 
 # Migrate CDB_TableMetadata
 cat >> ${output} <<'EOF'
@@ -29,32 +44,34 @@ BEGIN
   DROP FUNCTION public._CDB_UserQuotaInBytes();
 END;
 $$ LANGUAGE 'plpgsql';
-
--- Cartodbfy tables with a trigger using 'CDB_CheckQuota' or
--- 'CDB_TableMetadata_Trigger' from the 'public' schema
-select cartodb.CDB_CartodbfyTable(relname::regclass) from ( 
-  -- names of tables using public.CDB_CheckQuota or
-  -- public.CDB_TableMetadata_Trigger in their triggers
-  SELECT distinct c.relname
-  FROM
-    pg_trigger t,
-    pg_class c,
-    pg_proc p,
-    pg_namespace n
-  WHERE
-    n.nspname = 'public' AND
-    p.pronamespace = n.oid AND
-    p.proname IN ( 'cdb_checkquota', 'cdb_tablemetadata_trigger' ) AND
-    t.tgrelid = c.oid AND
-    p.oid = t.tgfoid 
-) as foo;
 EOF
 
-# Drop functions from public schema
-cat ${input} |
-  grep '^ *CREATE OR REPLACE FUNCTION' |
-  grep -v ' cartodb\.' | # should  only match DDL hooks
-  sed 's/).*$/);/' |
-  sed 's/DEFAULT [^ ,)]*//g' |
-  sed 's/CREATE OR REPLACE FUNCTION /DROP FUNCTION public./' |
-  cat >> ${output}
+## Cartodbfy tables with a trigger using 'CDB_CheckQuota' or
+## 'CDB_TableMetadata_Trigger' from the 'public' schema
+#cat >> ${output} <<'EOF'
+#select cartodb.CDB_CartodbfyTable(relname::regclass) from ( 
+#  -- names of tables using public.CDB_CheckQuota or
+#  -- public.CDB_TableMetadata_Trigger in their triggers
+#  SELECT distinct c.relname
+#  FROM
+#    pg_trigger t,
+#    pg_class c,
+#    pg_proc p,
+#    pg_namespace n
+#  WHERE
+#    n.nspname = 'public' AND
+#    p.pronamespace = n.oid AND
+#    p.proname IN ( 'cdb_checkquota', 'cdb_tablemetadata_trigger' ) AND
+#    t.tgrelid = c.oid AND
+#    p.oid = t.tgfoid 
+#) as foo;
+#EOF
+
+## Drop any leftover function from public schema (there should be none)
+#cat ${input} |
+#  grep '^ *CREATE OR REPLACE FUNCTION' |
+#  grep -v ' cartodb\.' | # should  only match DDL hooks
+#  sed 's/).*$/);/' |
+#  sed 's/DEFAULT [^ ,)]*//g' |
+#  sed 's/CREATE OR REPLACE FUNCTION /DROP FUNCTION IF EXISTS public./' |
+#  cat >> ${output}