Merge pull request #67 from CartoDB/faster-quota-check

Optimize CDB_UserDataSize (on behalf of @javisantana) #65

.gitignore

@@ -5,3 +5,4 @@ results/
 regression.*
 expected/test
 sql/test
+.idea/*

.travis.yml

@@ -4,7 +4,9 @@ addons:
   postgresql: 9.3
 
 before_install:
+  - sudo apt-get update
   #- sudo apt-get install -q postgresql-9.3-postgis-2.1
+  - sudo apt-get update
   - sudo apt-get install -q postgresql-server-dev-9.3
   - sudo apt-get install -q postgresql-plpython-9.3
   # Install schema_triggers

CONTRIBUTING.md

@@ -19,7 +19,7 @@ in which those scripts are loaded.
 
 Scripts would be best coded in a way to be usable both for creation
 and upgrade of the objects. This means using CREATE OR REPLACE for
-the functions, and whatever it takes to check existance of any previous
+the functions, and whatever it takes to check existence of any previous
 version of objects in other cases.
 
 When used as an extension (probably always from version 0.2.0 onwards)
@@ -27,8 +27,8 @@ all the objects will be installed in a "cartodb" schema. Take this into
 account to fully-qualify internal calls to avoid (possibly dangerous)
 name clashes.
 
-Every new feature (as well as bugfixes) should come with a testcase,
-see next session.
+Every new feature (as well as bugfixes) should come with a test case,
+see next section.
 
 Writing testcases
 -----------------
@@ -58,4 +58,3 @@ Starting with 0.2.0, the in-place reload can be done with an ad-hoc function:
 ```sql
 SELECT cartodb.cdb_extension_reload();
 ```
-

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.3.1
+EXTVERSION = 0.5.2
 
 SED = sed
 
@@ -20,6 +20,17 @@ UPGRADABLE = \
   0.2.0 \
   0.2.1 \
   0.3.0 \
+  0.3.0dev \
+  0.3.1 \
+  0.3.2 \
+  0.3.3 \
+  0.3.4 \
+  0.3.5 \
+  0.3.6 \
+  0.4.0 \
+  0.4.1 \
+  0.5.0 \
+  0.5.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -30,7 +41,6 @@ UPGRADES = \
      $(SED) 's/$$/--$(EXTVERSION).sql/' | \
      $(SED) 's/ /--$(EXTVERSION).sql $(EXTENSION)--/g')
 
-REV=$(shell git describe)
 GITDIR=$(shell test -d .git && echo '.git' || cat .git | $(SED) 's/^gitdir: //')
 
 DATA_built = \
@@ -72,7 +82,7 @@ $(EXTENSION).control: $(EXTENSION).control.in Makefile
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION) $(REV)/' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
 legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
@@ -95,7 +105,10 @@ legacy_regress: $(REGRESS_OLD) Makefile
 test_organization:
 	bash test/organization/test.sh
 
+test_extension_new:
+	bash test/extension/test.sh
+
 legacy_tests: legacy_regress 
 
-installcheck: legacy_tests test_organization
+installcheck: legacy_tests test_extension_new test_organization
 

NEWS.md

@@ -1,3 +1,46 @@
+0.5.2 (2015-01-29)
+------------------
+* Improvement: make CDB_UserDataSize functions much faster.
+
+0.5.1 (2014-11-21)
+------------------
+* Bugfix: Quota check and some organization permissions functions were not properly escaping table name.
+
+0.5.0 (2014-11-03)
+------------------
+* Support of raster tables for cartodbfication
+* Modified quota functions: vector tables stay the same, raster tables count as full size (as have no
+  the_geom + the_geom_webmercator combo) and raster overviews are not counted
+
+0.4.1 (2014-09-21)
+------------------
+* Bugfix for Cartodbfication: Set primary key of the table if not already present (e.g. tables created from SQL API)
+
+0.4.0 (2014-08-27)
+------------------
+Added CDB_Math_Mode function
+Changes in versioning: no revision is attached so it no longer uses `git describe` for the version.
+
+0.3.6 (2014-08-11)
+------------------
+Dummy release to solve some issues with cdb branch/tag
+
+0.3.5 (2014-08-11)
+------------------
+Inverting priority of CDB_CheckQuota qmax so gies more priority to existing user quota function over parameter value.
+
+0.3.4 (2014-08-01)
+------------------
+Fixes issue with schemas in CDB_QueryTables
+
+0.3.3 (2014-07-30)
+------------------
+* Splitting of CartodbfyTable method in subfunctions to be able to call in fragments and evade timeouts on hot zones
+
+0.3.2 (2014-07-28)
+------------------
+* Make 0.3.0dev version upgradeable
+
 0.3.1 (2014-07-22)
 ------------------
 * Dummy version. We start using semantic versioning

README.md

@@ -20,12 +20,16 @@ Dependencies
 Install
 -------
 
- make all install
+```sh
+make all install
+```
 
 Test installation
 -----------------
 
- make installcheck
+```sh
+make installcheck
+```
 
 NOTE: if ``test_ddl_triggers`` fails it's likely due to an incomplete
       installation of schema_triggers: you need to add ``schema_triggers.so``

expected/test_ddl_triggers.out

@@ -19,6 +19,12 @@ NOTICE:  event trigger "cdb_on_add_column" does not exist, skipping
 (1 row)
 
 create schema c;
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+ cdb_setuserquotainbytes 
+-------------------------
+                       0
+(1 row)
+
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';
@@ -247,7 +253,7 @@ FROM CDB_TableMetadata WHERE tabname = 'c.t3'::regclass;
 ----------------------------
 RESET SESSION AUTHORIZATION;
 drop schema c cascade;
-NOTICE:  drop cascades to 2 other objects
+NOTICE:  drop cascades to 3 other objects
 select count(*) from CDB_TableMetadata;
  count 
 -------

scripts-available/CDB_CartodbfyTable.sql

@@ -1,59 +1,40 @@
 -- Depends on:
+--   * CDB_ExtensionUtils.sql
 --   * CDB_TransformToWebmercator.sql
 --   * CDB_TableMetadata.sql
 --   * CDB_Quota.sql
 --   * _CDB_UserQuotaInBytes() function, installed by rails
---     (user.rebuild_quota_trigger, called by rake task
---      cartodb:db:update_test_quota_trigger)
+--     (user.rebuild_quota_trigger, called by rake task cartodb:db:update_test_quota_trigger)
 
--- Update the_geom_webmercator
-CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
-RETURNS trigger AS $$
-BEGIN
-  NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
-  RETURN NEW;
-END;
-$$ LANGUAGE plpgsql VOLATILE;
-
-CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
-RETURNS TRIGGER AS $$
-BEGIN
-   NEW.updated_at := now();
-   RETURN NEW;
-END;
-$$ LANGUAGE plpgsql VOLATILE;
-
--- Ensure a table is a "cartodb" table
--- See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table
-CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(schema_name TEXT, reloid REGCLASS)
-RETURNS void 
+-- 1) Required checks before running cartodbfication
+-- Either will pass silenty or raise an exception
+CREATE OR REPLACE FUNCTION _CDB_check_prerequisites(schema_name TEXT, reloid REGCLASS)
+RETURNS void
 AS $$
 DECLARE
   sql TEXT;
-  rec RECORD;
-  rec2 RECORD;
-  tabinfo RECORD;
-  had_column BOOLEAN;
-  i INTEGER;
-  new_name TEXT;
-  quota_in_bytes INT8;
-  exists_geom_cols BOOLEAN[];
 BEGIN
-
   IF cartodb.schema_exists(schema_name) = false THEN
     RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
   END IF;
 
   -- TODO: Check that user quota is set ?
   BEGIN
-  -- Content will be discarded
     EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name::text) INTO sql;
-  EXCEPTION WHEN undefined_function THEN
-    RAISE EXCEPTION 'Please set user quota before cartodbfying tables.';
+    EXCEPTION WHEN undefined_function THEN
+      RAISE EXCEPTION 'Please set user quota before cartodbfying tables.';
   END;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Drop cartodb triggers (might prevent changing columns)
 
+-- 2) Drop cartodb triggers (might prevent changing columns)
+CREATE OR REPLACE FUNCTION _CDB_drop_triggers(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
   -- "track_updates"
   sql := 'DROP TRIGGER IF EXISTS track_updates ON ' || reloid::text;
   EXECUTE sql;
@@ -71,10 +52,23 @@ BEGIN
   EXECUTE sql;
   sql := 'DROP TRIGGER IF EXISTS test_quota_per_row ON ' || reloid::text;
   EXECUTE sql;
+END;
+$$ LANGUAGE PLPGSQL;
 
-  -- Ensure required fields exist
 
-  -- We need a cartodb_id column
+-- 3) Cartodb_id creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_cartodb_id_column(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+  cartodb_id_name TEXT;
+BEGIN
   << cartodb_id_setup >>
   LOOP --{
     had_column := FALSE;
@@ -82,18 +76,17 @@ BEGIN
       sql := 'ALTER TABLE ' || reloid::text || ' ADD cartodb_id SERIAL NOT NULL UNIQUE';
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
+      cartodb_id_name := 'cartodb_id';
       EXIT cartodb_id_setup;
-    EXCEPTION
-    WHEN duplicate_column THEN
-      RAISE NOTICE 'Column cartodb_id already exists';
-      had_column := TRUE;
-    WHEN others THEN
-      RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
-        reloid, SQLERRM, SQLSTATE;
+      EXCEPTION
+      WHEN duplicate_column THEN
+        RAISE NOTICE 'Column cartodb_id already exists';
+        had_column := TRUE;
+      WHEN others THEN
+        RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
     END;
 
     IF had_column THEN
-
       SELECT pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id')
         AS seq INTO rec2;
 
@@ -101,9 +94,9 @@ BEGIN
       SELECT
         pg_catalog.pg_get_serial_sequence(reloid::text, 'cartodb_id') as seq,
         t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
-       WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
-         AND a.attname = 'cartodb_id'
+      WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = 'cartodb_id'
       INTO STRICT rec;
+
       -- 20=int2, 21=int4, 23=int8
       IF rec.oid NOT IN (20,21,23) THEN -- {
         RAISE NOTICE 'Existing cartodb_id field is of invalid type % (need int2, int4 or int8), renaming', rec.typname;
@@ -112,28 +105,27 @@ BEGIN
       ELSE -- }{
         sql := 'ALTER TABLE ' || reloid::text || ' ALTER COLUMN cartodb_id SET NOT NULL';
         IF NOT EXISTS ( SELECT c.conname FROM pg_constraint c, pg_attribute a
-                      WHERE c.conkey = ARRAY[a.attnum] AND c.conrelid = reloid
-                        AND a.attrelid = reloid
-                        AND NOT a.attisdropped
-                        AND a.attname = 'cartodb_id'
-                        AND c.contype IN ( 'u', 'p' ) ) -- unique or pkey 
+        WHERE c.conkey = ARRAY[a.attnum] AND c.conrelid = reloid
+              AND a.attrelid = reloid
+              AND NOT a.attisdropped
+              AND a.attname = 'cartodb_id'
+              AND c.contype IN ( 'u', 'p' ) ) -- unique or pkey
         THEN
           sql := sql || ', ADD unique(cartodb_id)';
         END IF;
         BEGIN
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
+          cartodb_id_name := 'cartodb_id';
           EXIT cartodb_id_setup;
-        EXCEPTION
-        WHEN unique_violation OR not_null_violation THEN
-          RAISE NOTICE '%, renaming', SQLERRM;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)',
-            reloid, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN unique_violation OR not_null_violation THEN
+            RAISE NOTICE '%, renaming', SQLERRM;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
         END;
       END IF; -- }
 
-
       -- invalid column, need rename and re-create it
       i := 0;
       << rename_column >>
@@ -143,18 +135,17 @@ BEGIN
           sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN cartodb_id TO ' || new_name;
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
-        EXCEPTION
-        WHEN duplicate_column THEN
-          i := i+1;
-          CONTINUE rename_column;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)',
-            reloid, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN duplicate_column THEN
+            i := i+1;
+            CONTINUE rename_column;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (renaming cartodb_id): % (%)', reloid, SQLERRM, SQLSTATE;
         END;
+        cartodb_id_name := new_name;
         EXIT rename_column;
       END LOOP; --}
       CONTINUE cartodb_id_setup;
-
     END IF;
   END LOOP; -- }
 
@@ -163,12 +154,11 @@ BEGIN
     RAISE NOTICE 'Trying to recover data from % column', new_name;
     BEGIN
       -- Copy existing values to new field
-      -- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-      --       is also known to be faster for tables with many rows
+      -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
       -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
       sql := 'ALTER TABLE ' || reloid::text
-          || ' ALTER cartodb_id TYPE int USING '
-          || new_name || '::int4';
+             || ' ALTER cartodb_id TYPE int USING '
+             || new_name || '::int4';
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
@@ -183,23 +173,51 @@ BEGIN
 
       -- Reset sequence name
       sql := 'ALTER SEQUENCE ' || rec2.seq::text
-          || ' RESTART WITH ' || rec.max + 1;
+             || ' RESTART WITH ' || rec.max + 1;
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
-      -- Drop old column (all went find if we got here)
+      -- Drop old column (all went fine if we got here)
       sql := 'ALTER TABLE ' || reloid::text || ' DROP ' || new_name;
       RAISE DEBUG 'Running %', sql;
       EXECUTE sql;
 
-    EXCEPTION
-    WHEN others THEN
-      RAISE NOTICE 'Could not initialize cartodb_id with existing values: % (%)',
+      EXCEPTION
+      WHEN others THEN
+        RAISE NOTICE 'Could not initialize cartodb_id with existing values: % (%)',
         SQLERRM, SQLSTATE;
     END;
   END IF;
 
-  -- We need created_at and updated_at
+  -- Set primary key of the table if not already present (e.g. tables created from SQL API)
+  IF cartodb_id_name IS NULL THEN
+    RAISE EXCEPTION 'Cartodbfying % (Didnt get cartodb_id field name)', reloid;
+  END IF;
+  BEGIN
+    sql := 'ALTER TABLE ' || reloid::text || ' ADD PRIMARY KEY (cartodb_id)';
+    EXECUTE sql;
+    EXCEPTION
+    WHEN others THEN
+      RAISE DEBUG 'Table % Already had PRIMARY KEY', reloid;
+  END;
+
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 4) created_at and updated_at creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_timestamp_columns(reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+BEGIN
+
   FOR rec IN SELECT * FROM ( VALUES ('created_at'), ('updated_at') ) t(cname)
   LOOP --{
     new_name := null;
@@ -208,48 +226,43 @@ BEGIN
       had_column := FALSE;
       BEGIN
         sql := 'ALTER TABLE ' || reloid::text || ' ADD ' || rec.cname
-          || ' TIMESTAMPTZ NOT NULL DEFAULT now()';
+               || ' TIMESTAMPTZ NOT NULL DEFAULT now()';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
         EXIT column_setup;
-      EXCEPTION
-      WHEN duplicate_column THEN
-        RAISE NOTICE 'Column % already exists', rec.cname;
-        had_column := TRUE;
-      WHEN others THEN
-        RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-          reloid, rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN duplicate_column THEN
+          RAISE NOTICE 'Column % already exists', rec.cname;
+          had_column := TRUE;
+        WHEN others THEN
+          RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
       END;
 
       IF had_column THEN
-
         -- Check data type is a TIMESTAMP WITH TIMEZONE
         SELECT t.typname, t.oid, a.attnotnull FROM pg_type t, pg_attribute a
-         WHERE a.atttypid = t.oid AND a.attrelid = reloid
-           AND NOT a.attisdropped AND a.attname = rec.cname
+        WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
         INTO STRICT rec2;
         IF rec2.oid NOT IN (1184) THEN -- timestamptz {
-          RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.
-cname, rec2.typname;
+          RAISE NOTICE 'Existing % field is of invalid type % (need timestamptz), renaming', rec.cname, rec2.typname;
         ELSE -- }{
           -- Ensure data type is a TIMESTAMP WITH TIMEZONE
           sql := 'ALTER TABLE ' || reloid::text
-            || ' ALTER ' || rec.cname
-            || ' SET NOT NULL,'
-            || ' ALTER ' || rec.cname
-            || ' SET DEFAULT now()';
+                 || ' ALTER ' || rec.cname
+                 || ' SET NOT NULL,'
+                 || ' ALTER ' || rec.cname
+                 || ' SET DEFAULT now()';
           BEGIN
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
             EXIT column_setup;
-          EXCEPTION
-          WHEN not_null_violation THEN -- failed not-null
-            RAISE NOTICE '%, renaming', SQLERRM;
-          WHEN cannot_coerce THEN -- failed cast
-            RAISE NOTICE '%, renaming', SQLERRM;
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-              reloid, rec.cname, SQLERRM, SQLSTATE;
+            EXCEPTION
+            WHEN not_null_violation THEN -- failed not-null
+              RAISE NOTICE '%, renaming', SQLERRM;
+            WHEN cannot_coerce THEN -- failed cast
+              RAISE NOTICE '%, renaming', SQLERRM;
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
         END IF; -- }
 
@@ -262,18 +275,17 @@ cname, rec2.typname;
             sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN ' || rec.cname || ' TO ' || new_name;
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
-          EXCEPTION
-          WHEN duplicate_column THEN
-            i := i+1;
-            CONTINUE rename_column;
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (renaming %): % (%)',
+            EXCEPTION
+            WHEN duplicate_column THEN
+              i := i+1;
+              CONTINUE rename_column;
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (renaming %): % (%)',
               reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
           EXIT rename_column;
         END LOOP; --}
         CONTINUE column_setup;
-
       END IF;
     END LOOP; -- }
 
@@ -282,12 +294,11 @@ cname, rec2.typname;
       RAISE NOTICE 'Trying to recover data from % coumn', new_name;
       BEGIN
         -- Copy existing values to new field
-        -- NOTE: using ALTER is a workaround to a PostgreSQL bug and
-        --       is also known to be faster for tables with many rows
+        -- NOTE: using ALTER is a workaround to a PostgreSQL bug and is also known to be faster for tables with many rows
         -- See http://www.postgresql.org/message-id/20140530143150.GA11051@localhost
         sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
-            || ' TYPE TIMESTAMPTZ USING '
-            || new_name || '::timestamptz';
+               || ' TYPE TIMESTAMPTZ USING '
+               || new_name || '::timestamptz';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
 
@@ -296,21 +307,36 @@ cname, rec2.typname;
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
 
-      EXCEPTION
-      WHEN others THEN
-        RAISE NOTICE 'Could not initialize % with existing values: % (%)',
-          rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN others THEN
+          RAISE NOTICE 'Could not initialize % with existing values: % (%)', rec.cname, SQLERRM, SQLSTATE;
       END;
     END IF; -- }
-
   END LOOP; -- }
 
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 5) the_geom and the_geom_webmercator creation & validation or renaming if invalid
+CREATE OR REPLACE FUNCTION _CDB_create_the_geom_columns(reloid REGCLASS)
+  RETURNS BOOLEAN[]
+AS $$
+DECLARE
+  sql TEXT;
+  rec RECORD;
+  rec2 RECORD;
+  had_column BOOLEAN;
+  i INTEGER;
+  new_name TEXT;
+  exists_geom_cols BOOLEAN[];
+BEGIN
   -- We need the_geom and the_geom_webmercator
   FOR rec IN SELECT * FROM ( VALUES ('the_geom',4326), ('the_geom_webmercator',3857) ) t(cname,csrid) LOOP --{
     << column_setup >> LOOP --{
       BEGIN
         sql := 'ALTER TABLE ' || reloid::text || ' ADD ' || rec.cname
-          || ' GEOMETRY(geometry,' || rec.csrid || ')';
+               || ' GEOMETRY(geometry,' || rec.csrid || ')';
         RAISE DEBUG 'Running %', sql;
         EXECUTE sql;
         sql := 'CREATE INDEX ON ' || reloid::text || ' USING GIST ( ' || rec.cname || ')';
@@ -318,25 +344,22 @@ cname, rec2.typname;
         EXECUTE sql;
         exists_geom_cols := array_append(exists_geom_cols, false);
         EXIT column_setup;
-      EXCEPTION
-      WHEN duplicate_column THEN
-        exists_geom_cols := array_append(exists_geom_cols, true);
-        RAISE NOTICE 'Column % already exists', rec.cname;
-      WHEN others THEN
-        RAISE EXCEPTION 'Cartodbfying % (%): % (%)',
-          reloid, rec.cname, SQLERRM, SQLSTATE;
+        EXCEPTION
+        WHEN duplicate_column THEN
+          exists_geom_cols := array_append(exists_geom_cols, true);
+          RAISE NOTICE 'Column % already exists', rec.cname;
+        WHEN others THEN
+          RAISE EXCEPTION 'Cartodbfying % (%): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
       END;
 
       << column_fixup >>
       LOOP --{
-
         -- Check data type is a GEOMETRY
         SELECT t.typname, t.oid, a.attnotnull,
-               postgis_typmod_srid(a.atttypmod) as srid,
-               postgis_typmod_type(a.atttypmod) as gtype
-         FROM pg_type t, pg_attribute a
-         WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped
-           AND a.attname = rec.cname
+          postgis_typmod_srid(a.atttypmod) as srid,
+          postgis_typmod_type(a.atttypmod) as gtype
+        FROM pg_type t, pg_attribute a
+        WHERE a.atttypid = t.oid AND a.attrelid = reloid AND NOT a.attisdropped AND a.attname = rec.cname
         INTO STRICT rec2;
 
         IF rec2.typname NOT IN ('geometry') THEN -- {
@@ -345,46 +368,44 @@ cname, rec2.typname;
         END IF; -- }
 
         IF rec2.srid != rec.csrid THEN -- {
-            BEGIN
-              sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
-                || ' TYPE geometry(' || rec2.gtype || ',' || rec.csrid || ') USING ST_Transform('
-                || rec.cname || ',' || rec.csrid || ')';
-              RAISE DEBUG 'Running %', sql;
-              EXECUTE sql;
+          BEGIN
+            sql := 'ALTER TABLE ' || reloid::text || ' ALTER ' || rec.cname
+                   || ' TYPE geometry(' || rec2.gtype || ',' || rec.csrid || ') USING ST_Transform('
+                   || rec.cname || ',' || rec.csrid || ')';
+            RAISE DEBUG 'Running %', sql;
+            EXECUTE sql;
             EXCEPTION
             WHEN others THEN
               RAISE NOTICE 'Could not enforce SRID % to column %: %, renaming', rec.csrid, rec.cname, SQLERRM;
               EXIT column_fixup; -- cannot fix, will rename
-            END;
+          END;
         END IF; -- }
 
         -- add gist indices if not there already
         IF NOT EXISTS ( SELECT ir.relname
                         FROM pg_am am, pg_class ir,
-                             pg_class c, pg_index i,
-                             pg_attribute a
+                          pg_class c, pg_index i,
+                          pg_attribute a
                         WHERE c.oid  = reloid AND i.indrelid = c.oid
-                          AND a.attname = rec.cname
-                          AND i.indexrelid = ir.oid AND i.indnatts = 1
-                          AND i.indkey[0] = a.attnum AND a.attrelid = c.oid
-                          AND NOT a.attisdropped AND am.oid = ir.relam
-                          AND am.amname = 'gist' )
+                              AND a.attname = rec.cname
+                              AND i.indexrelid = ir.oid AND i.indnatts = 1
+                              AND i.indkey[0] = a.attnum AND a.attrelid = c.oid
+                              AND NOT a.attisdropped AND am.oid = ir.relam
+                              AND am.amname = 'gist' )
         THEN -- {
           BEGIN
             sql := 'CREATE INDEX ON ' || reloid::text || ' USING GIST ( ' || rec.cname || ')';
             RAISE DEBUG 'Running %', sql;
             EXECUTE sql;
-          EXCEPTION
-          WHEN others THEN
-            RAISE EXCEPTION 'Cartodbfying % (% index): % (%)',
-              reloid, rec.cname, SQLERRM, SQLSTATE;
+            EXCEPTION
+            WHEN others THEN
+              RAISE EXCEPTION 'Cartodbfying % (% index): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
           END;
         END IF; -- }
 
         -- if we reached this line, all went good
         EXIT column_setup;
-
-      END LOOP; -- } column_fixup 
+      END LOOP; -- } column_fixup
 
       -- invalid column, need rename and re-create it
       i := 0;
@@ -395,87 +416,236 @@ cname, rec2.typname;
           sql := 'ALTER TABLE ' || reloid::text || ' RENAME COLUMN ' || rec.cname || ' TO ' || new_name;
           RAISE DEBUG 'Running %', sql;
           EXECUTE sql;
-        EXCEPTION
-        WHEN duplicate_column THEN
-          i := i+1;
-          CONTINUE rename_column;
-        WHEN others THEN
-          RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)',
-            reloid, rec.cname, SQLERRM, SQLSTATE;
+          EXCEPTION
+          WHEN duplicate_column THEN
+            i := i+1;
+            CONTINUE rename_column;
+          WHEN others THEN
+            RAISE EXCEPTION 'Cartodbfying % (rename %): % (%)', reloid, rec.cname, SQLERRM, SQLSTATE;
         END;
         EXIT rename_column;
       END LOOP; --}
       CONTINUE column_setup;
-
-    END LOOP; -- } column_setup 
-
+    END LOOP; -- } column_setup
   END LOOP; -- } on expected geometry columns
 
-  -- Initialize the_geom with values from the_geom_webmercator
-  -- do this only if the_geom_webmercator was found (not created)
-  -- _and_ the_geom was NOT found.
-  IF exists_geom_cols[2] AND NOT exists_geom_cols[1] THEN
-    sql := 'UPDATE ' || reloid::text || ' SET the_geom = ST_Transform(the_geom_webmercator, 4326) ';
-    EXECUTE sql;
-  END IF;
-
-  -- Initialize the_geom_webmercator with values from the_geom
-  -- do this only if the_geom was found (not created)
-  -- _and_ the_geom_webmercator was NOT found.
-  IF exists_geom_cols[1] AND NOT exists_geom_cols[2] THEN
-    sql := 'UPDATE ' || reloid::text || ' SET the_geom_webmercator = public.CDB_TransformToWebmercator(the_geom) ';
-    EXECUTE sql;
-  END IF;
-
-  -- Re-create all triggers
-
-  -- NOTE: drop/create has the side-effect of re-enabling disabled triggers
-
-  -- "track_updates"
-  sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
-      || reloid::text
-      || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
-  EXECUTE sql;
-
-  -- "update_the_geom_webmercator"
-  -- TODO: why _before_ and not after ?
-  sql := 'CREATE trigger update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()';
-  EXECUTE sql;
-
-  -- "update_updated_at"
-  -- TODO: why _before_ and not after ?
-  sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
-  EXECUTE sql;
-
-  -- "test_quota" and "test_quota_per_row"
-
-  sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
-      || reloid::text
-      || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
-      || schema_name::text
-      || ''')';
-  EXECUTE sql;
-
-  sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON '
-      || reloid::text
-      || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', '''
-      || schema_name::text
-      || ''')';
-  EXECUTE sql;
- 
+  RETURN exists_geom_cols;
 END;
 $$ LANGUAGE PLPGSQL;
 
 
+-- 6) Initialize the_geom with values from the_geom_webmercator
+-- do this only if the_geom_webmercator was found (not created) and the_geom was NOT found.
+CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_from_the_geom_webmercator(reloid REGCLASS, geom_columns_exist BOOLEAN[])
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+  IF geom_columns_exist[2] AND NOT geom_columns_exist[1] THEN
+    sql := 'UPDATE ' || reloid::text || ' SET the_geom = ST_Transform(the_geom_webmercator, 4326) ';
+    EXECUTE sql;
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 7) Initialize the_geom_webmercator with values from the_geom
+-- do this only if the_geom was found (not created) and the_geom_webmercator was NOT found.
+CREATE OR REPLACE FUNCTION _CDB_populate_the_geom_webmercator_from_the_geom(reloid REGCLASS, geom_columns_exist BOOLEAN[])
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+  IF geom_columns_exist[1] AND NOT geom_columns_exist[2] THEN
+    sql := 'UPDATE ' || reloid::text || ' SET the_geom_webmercator = public.CDB_TransformToWebmercator(the_geom) ';
+    EXECUTE sql;
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- 8.a) Create all triggers
+-- NOTE: drop/create has the side-effect of re-enabling disabled triggers
+CREATE OR REPLACE FUNCTION _CDB_create_triggers(schema_name TEXT, reloid REGCLASS)
+RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+-- "track_updates"
+  sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
+         || reloid::text
+         || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
+  EXECUTE sql;
+
+-- "update_the_geom_webmercator"
+-- TODO: why _before_ and not after ?
+  sql := 'CREATE trigger update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_the_geom_webmercator()';
+  EXECUTE sql;
+
+-- "update_updated_at"
+-- TODO: why _before_ and not after ?
+  sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
+  EXECUTE sql;
+
+-- "test_quota" and "test_quota_per_row"
+
+  sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+
+  sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- 8.b) Create all raster triggers
+-- NOTE: drop/create has the side-effect of re-enabling disabled triggers
+CREATE OR REPLACE FUNCTION _CDB_create_raster_triggers(schema_name TEXT, reloid REGCLASS)
+  RETURNS void
+AS $$
+DECLARE
+  sql TEXT;
+BEGIN
+-- "track_updates"
+  sql := 'CREATE trigger track_updates AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON '
+         || reloid::text
+         || ' FOR EACH STATEMENT EXECUTE PROCEDURE public.cdb_tablemetadata_trigger()';
+  EXECUTE sql;
+
+-- "update_updated_at"
+-- TODO: why _before_ and not after ?
+  sql := 'CREATE trigger update_updated_at_trigger BEFORE UPDATE ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public._CDB_update_updated_at()';
+  EXECUTE sql;
+
+-- "test_quota" and "test_quota_per_row"
+
+  sql := 'CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' EXECUTE PROCEDURE public.CDB_CheckQuota(1, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+
+  sql := 'CREATE TRIGGER test_quota_per_row BEFORE UPDATE OR INSERT ON '
+         || reloid::text
+         || ' FOR EACH ROW EXECUTE PROCEDURE public.CDB_CheckQuota(0.001, ''-1'', '''
+         || schema_name::text
+         || ''')';
+  EXECUTE sql;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+
+-- Update the_geom_webmercator
+CREATE OR REPLACE FUNCTION _CDB_update_the_geom_webmercator()
+  RETURNS trigger
+AS $$
+BEGIN
+  NEW.the_geom_webmercator := public.CDB_TransformToWebmercator(NEW.the_geom);
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+CREATE OR REPLACE FUNCTION _CDB_update_updated_at()
+  RETURNS TRIGGER AS $$
+BEGIN
+   NEW.updated_at := now();
+   RETURN NEW;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+
+-- Auxiliary function
+CREATE OR REPLACE FUNCTION cartodb._CDB_is_raster_table(schema_name TEXT, reloid REGCLASS)
+  RETURNS BOOLEAN
+AS $$
+DECLARE
+  sql TEXT;
+  is_raster BOOLEAN;
+  rel_name TEXT;
+BEGIN
+  IF cartodb.schema_exists(schema_name) = FALSE THEN
+    RAISE EXCEPTION 'Invalid schema name "%"', schema_name;
+  END IF;
+
+  SELECT relname FROM pg_class WHERE oid=reloid INTO rel_name;
+
+  BEGIN
+    sql := 'SELECT the_raster_webmercator FROM '
+          || quote_ident(schema_name::TEXT)
+          || '.'
+          || quote_ident(rel_name::TEXT)
+          || ' LIMIT 1';
+    is_raster = TRUE;
+    EXECUTE sql;
+
+    EXCEPTION WHEN undefined_column THEN
+      is_raster = FALSE;
+  END;
+
+  RETURN is_raster;
+END;
+$$ LANGUAGE PLPGSQL;
+
+
+
+-- ////////////////////////////////////////////////////
+
+-- Ensure a table is a "cartodb" table (See https://github.com/CartoDB/cartodb/wiki/CartoDB-user-table)
+-- Rails code replicates this call at User.cartodbfy()
+CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(schema_name TEXT, reloid REGCLASS)
+RETURNS void 
+AS $$
+DECLARE
+  exists_geom_cols BOOLEAN[];
+  is_raster BOOLEAN;
+BEGIN
+
+  PERFORM cartodb._CDB_check_prerequisites(schema_name, reloid);
+
+  PERFORM cartodb._CDB_drop_triggers(reloid);
+
+  -- Ensure required fields exist
+  PERFORM cartodb._CDB_create_cartodb_id_column(reloid);
+  PERFORM cartodb._CDB_create_timestamp_columns(reloid);
+
+  SELECT cartodb._CDB_is_raster_table(schema_name, reloid) INTO is_raster;
+  IF is_raster THEN
+    PERFORM cartodb._CDB_create_raster_triggers(schema_name, reloid);
+  ELSE
+    SELECT cartodb._CDB_create_the_geom_columns(reloid) INTO exists_geom_cols;
+
+    -- Both only populate if proceeds
+    PERFORM cartodb._CDB_populate_the_geom_from_the_geom_webmercator(reloid, exists_geom_cols);
+    PERFORM cartodb._CDB_populate_the_geom_webmercator_from_the_geom(reloid, exists_geom_cols);
+
+    PERFORM cartodb._CDB_create_triggers(schema_name, reloid);
+  END IF;
+
+END;
+$$ LANGUAGE PLPGSQL;
+
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(reloid REGCLASS)
 RETURNS void
 AS $$
 BEGIN
-  PERFORM public.CDB_CartodbfyTable('public', reloid);
+  PERFORM cartodb.CDB_CartodbfyTable('public', reloid);
 END;
-$$
-LANGUAGE PLPGSQL;
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_DDLTriggers.sql

@@ -4,14 +4,17 @@ CREATE OR REPLACE FUNCTION cartodb.cdb_handle_create_table ()
 RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_relation_create_eventinfo();
 
   -- We're only interested in real relations
   IF (event_info.new).relkind != 'r' THEN RETURN; END IF;
 
-  RAISE DEBUG 'Relation % of kind % created in namespace oid %',
-	 event_info.relation, (event_info.new).relkind, (event_info.new).relnamespace;
+  SELECT nspname FROM pg_namespace WHERE oid=(event_info.new).relnamespace INTO rel_namespace;
+
+  RAISE DEBUG 'Relation % of kind % created in table % namespace % (oid %)',
+	 event_info.relation, (event_info.new).relkind, (event_info.new).relname::TEXT, rel_namespace, (event_info.new).relnamespace;
 
   -- We don't want to react to alters triggered by superuser,
   IF current_setting('is_superuser') = 'on' THEN
@@ -22,7 +25,7 @@ BEGIN
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
   -- CDB_CartodbfyTable must not create tables, or infinite loop will happen
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
@@ -63,6 +66,7 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_column_alter_eventinfo();
 
@@ -80,9 +84,11 @@ BEGIN
     RETURN;
   END IF;
 
+  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
+
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 
@@ -100,6 +106,7 @@ RETURNS event_trigger SECURITY DEFINER LANGUAGE plpgsql AS $$
 DECLARE
   event_info RECORD;
   rel RECORD;
+  rel_namespace TEXT;
 BEGIN
   event_info := schema_triggers.get_column_drop_eventinfo();
 
@@ -117,9 +124,11 @@ BEGIN
     RETURN;
   END IF;
 
+  SELECT nspname FROM pg_namespace WHERE oid = rel.relnamespace INTO rel_namespace;
+
   PERFORM cartodb.cdb_disable_ddl_hooks();
 
-  PERFORM cartodb.CDB_CartodbfyTable(event_info.relation);
+  PERFORM cartodb.CDB_CartodbfyTable(rel_namespace, event_info.relation);
 
   PERFORM cartodb.cdb_enable_ddl_hooks();
 

scripts-available/CDB_Math.sql

@@ -0,0 +1,26 @@
+-- CartoDB Math SQL functions
+
+
+-- Mode
+-- https://wiki.postgresql.org/wiki/Aggregate_Mode
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Math_final_mode(anyarray)
+  RETURNS anyelement AS
+$BODY$
+    SELECT a
+    FROM unnest($1) a
+    GROUP BY 1 
+    ORDER BY COUNT(1) DESC, 1
+    LIMIT 1;
+$BODY$
+LANGUAGE 'sql' IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS cartodb.CDB_Math_Mode(anyelement);
+
+CREATE AGGREGATE cartodb.CDB_Math_Mode(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=_CDB_Math_final_mode,
+  INITCOND='{}'
+);
+

scripts-available/CDB_Organizations.sql

@@ -35,7 +35,7 @@ FUNCTION cartodb.CDB_Organization_Add_Table_Read_Permission(from_schema text, ta
 AS $$
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
-    EXECUTE 'GRANT SELECT ON "' || from_schema || '".' || table_name || ' TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
@@ -54,7 +54,7 @@ FUNCTION cartodb.CDB_Organization_Add_Table_Read_Write_Permission(from_schema te
 AS $$
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
-    EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '".' || table_name || ' TO "' || to_role_name || '"';
+    EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
@@ -73,7 +73,7 @@ FUNCTION cartodb.CDB_Organization_Remove_Access_Permission(from_schema text, tab
     RETURNS void
 AS $$
 BEGIN
-    EXECUTE 'REVOKE ALL PRIVILEGES ON TABLE "' || from_schema || '".' || table_name || ' FROM "' || to_role_name || '"';
+    EXECUTE 'REVOKE ALL PRIVILEGES ON TABLE "' || from_schema || '"."' || table_name || '" FROM "' || to_role_name || '"';
     -- EXECUTE 'REVOKE USAGE ON SCHEMA ' || from_schema || ' FROM "' || to_role_name || '"';
     -- We need to revoke usage on schema only if we are revoking privileges from the last table where to_role_name has
     -- any permission granted within the schema from_schema

scripts-available/CDB_QueryTables.sql

@@ -39,7 +39,7 @@ BEGIN
       inp AS (
         SELECT
           xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
-          xpath('//x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
+          xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
       )
       SELECT unnest(x)::name as p, unnest(s)::name as sc from inp
     LOOP

scripts-available/CDB_Quota.sql

@@ -2,20 +2,45 @@
 CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
 RETURNS bigint AS
 $$
-  -- TODO: double check this query. Maybe use CDB_TableMetadata for lookup ?
-  --       also, it's "table_name" sounds sensible to search_path
-  --
-  -- NOTE: division by 2 is an hack for the_geom_webmercator
-  --
-  SELECT coalesce(int8(sum(pg_total_relation_size(schema_name || '.' || table_name)) / 2), 0)
-    AS quota
-  FROM information_schema.tables
-  WHERE table_catalog = current_database() AND table_schema = schema_name
+DECLARE
+  total_size INT8;
+BEGIN
+  WITH raster_tables AS (
+    SELECT o_table_name, r_table_name FROM raster_overviews
+      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
+  ),
+  user_tables AS (
+    SELECT table_name FROM information_schema.tables
+      WHERE table_catalog = current_database() AND table_schema = schema_name
         AND table_name != 'spatial_ref_sys'
         AND table_name != 'cdb_tablemetadata'
-        AND table_type = 'BASE TABLE';
+        AND table_type = 'BASE TABLE'
+  ),
+  table_cat AS (
+    SELECT
+      table_name,
+      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
+    FROM user_tables
+  ),
+  sizes AS (
+    SELECT COALESCE(INT8(SUM(pg_total_relation_size('"' || schema_name || '"."' || table_name || '"')))) table_size,
+      CASE
+        WHEN is_overview THEN 0
+	WHEN is_raster THEN 1
+	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
+      END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
+  )
+  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
+
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
 $$
-LANGUAGE 'sql' VOLATILE;
+LANGUAGE 'plpgsql' VOLATILE;
 
 
 -- Return the estimated size of user data. Used for quota checking.
@@ -46,16 +71,20 @@ BEGIN
   ELSE
     schema_name := 'public';
   END IF;
-  -- Hack to support old versions of CDB_CheckQuota with 2 params but without schema_name
-  IF TG_NARGS >= 2 AND TG_ARGV[1] <> '-1' THEN
-    qmax := TG_ARGV[1];
-  ELSE
+
+  -- By default try to use quota function, and if not present then rely on the one specified by params
+  BEGIN
+    EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax;
+  EXCEPTION WHEN undefined_function THEN
     BEGIN
-      EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax;
-      EXCEPTION WHEN undefined_function THEN
-      RAISE EXCEPTION 'Missing "%"._CDB_UserQuotaInBytes()', schema_name;
+      IF TG_NARGS >= 2 AND TG_ARGV[1] <> '-1' THEN
+        qmax := TG_ARGV[1];
+      ELSE
+        RAISE EXCEPTION 'Missing "%"._CDB_UserQuotaInBytes()', schema_name;
+      END IF;
     END;
-  END IF;
+  END;
+
   pbfact := TG_ARGV[0];
 
   dice := random();

scripts-enabled/220-CDB_Math.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Math.sql

sql/test_ddl_triggers.sql

@@ -8,6 +8,8 @@ SELECT cartodb.cdb_enable_ddl_hooks();
 
 create schema c;
 
+SELECT CDB_SetUserQuotaInBytes('c', 0);
+
 CREATE USER cartodb_postgresql_unpriv_user;
 GRANT ALL ON SCHEMA c to cartodb_postgresql_unpriv_user;
 SET SESSION AUTHORIZATION 'cartodb_postgresql_unpriv_user';

test/CDB_CartodbfyTableTest_expect

@@ -2,6 +2,8 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
+CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(schema_name, reloid)"
+PL/pgSQL function cdb_cartodbfytable(text,regclass) line 7 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE

test/CDB_MathTest.sql

@@ -0,0 +1,4 @@
+
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1,2,3]) a;
+SELECT cdb_math_mode(a) from unnest(ARRAY[1]) a;

test/CDB_MathTest_expect

@@ -0,0 +1,3 @@
+2
+1
+1

test/CDB_QuotaTest_expect

@@ -7,7 +7,7 @@ ERROR:  Quota exceeded by 3.9990234375KB
 
 INSERT 0 1024
 8
-ERROR:  Quota exceeded by 103.9921875KB
+ERROR:  Quota exceeded by 123.9921875KB
 0
 INSERT 0 1
 DROP TABLE

test/extension/run_at_cartodb_schema.sql

@@ -0,0 +1,3 @@
+SET SCHEMA 'cartodb';
+\i scripts-available/CDB_Quota.sql
+SET SCHEMA 'public';

test/extension/test.sh

@@ -0,0 +1,284 @@
+#!/bin/sh
+
+#
+# Tests for the extension since version 0.5.0. They don't replace SQL based ones, for now need to run both
+#
+
+# It is expected that you run this script as a PostgreSQL superuser, for example:
+#
+#   PGUSER=postgres bash ./test.sh
+#
+
+DATABASE=test_extension
+CMD='echo psql'
+CMD=psql
+
+OK=0
+PARTIALOK=0
+
+function set_failed() {
+    OK=1
+    PARTIALOK=1
+}
+
+
+function clear_partial_result() {
+    PARTIALOK=0
+}
+
+
+function sql() {
+    local ROLE
+    local QUERY
+    if [[ $# -ge 2 ]]
+    then
+        ROLE="$1"
+        QUERY="$2"
+    else
+        QUERY="$1"
+    fi
+
+    if [ -n "${ROLE}" ]; then
+      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+    else
+      log_debug "Executing query '${QUERY}'"
+      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+    fi
+    CODERESULT=$?
+
+    echo ${RESULT}
+    echo
+
+    if [[ ${CODERESULT} -ne 0 ]]
+    then
+        echo -n "FAILED TO EXECUTE QUERY: "
+        log_warning "${QUERY}"
+        if [[ "$3" != "fails" ]]
+        then
+            log_error "${QUERY}"
+            set_failed
+        fi
+    else
+        if [[ "$3" == "fails" ]]
+        then
+            log_error "QUERY: '${QUERY}' was expected to fail and it did not fail"
+            set_failed
+        fi
+    fi
+
+    if [[ "$3" == "should" ]]
+    then
+        if [[ "${RESULT}" != "$4" ]]
+        then
+            log_error "QUERY '${QUERY}' expected result '${4}' but got '${RESULT}'"
+            set_failed
+        fi
+    fi
+}
+
+
+function log_info()
+{
+    echo
+    echo
+    echo
+    _log "1;34m" "$1"
+}
+
+function log_error() {
+    _log "1;31m" "$1"
+}
+
+function log_debug() {
+    _log "1;32m" "> $1"
+}
+
+function log_warning() {
+    _log "0;33m" "$1"
+}
+
+function _log() {
+    echo -e "\033[$1$2\033[0m"
+}
+
+# '############################ HELPERS #############################'
+function create_role_and_schema() {
+    local ROLE=$1
+    sql "CREATE ROLE ${ROLE} LOGIN;"
+    sql "GRANT CONNECT ON DATABASE \"${DATABASE}\" TO ${ROLE};"
+    sql "CREATE SCHEMA ${ROLE} AUTHORIZATION ${ROLE};"
+    sql "SELECT cartodb.CDB_Organization_Create_Member('${ROLE}');"
+}
+
+
+function drop_role_and_schema() {
+    local ROLE=$1
+    sql "DROP SCHEMA \"${ROLE}\";"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM \"${ROLE}\";"
+    sql "DROP ROLE \"${ROLE}\";"
+}
+
+
+function create_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} ( a int );"
+}
+
+
+function create_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "create_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local RASTER_COL="the_raster_webmercator"
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "CREATE TABLE ${ROLE}.${OVERVIEW_TABLENAME} (rid serial PRIMARY KEY, ${RASTER_COL} raster);"
+
+    sql ${ROLE} "SELECT AddOverviewConstraints('${ROLE}','${OVERVIEW_TABLENAME}','${RASTER_COL}','${ROLE}','${TABLENAME}','${RASTER_COL}',2);"
+}
+
+function drop_raster_table() {
+    if [[ $# -ne 2 ]]
+    then
+        log_error "drop_raster_table requires two arguments: role and table_name"
+        exit 1
+    fi
+    local ROLE="$1"
+    local TABLENAME="$2"
+    local OVERVIEW_TABLENAME="o_2_${TABLENAME}"
+
+    sql ${ROLE} "DROP TABLE ${ROLE}.${OVERVIEW_TABLENAME};"
+    sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
+}
+
+
+function setup() {
+    ${CMD} -c "CREATE DATABASE ${DATABASE}"
+    sql "CREATE SCHEMA cartodb;"
+    sql "GRANT USAGE ON SCHEMA cartodb TO public;"
+    sql "CREATE EXTENSION postgis;"
+
+    log_info "########################### BOOTSTRAP ###########################"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
+    ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
+
+
+    log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_testmember_1
+    create_role_and_schema cdb_testmember_2
+
+    create_table cdb_testmember_1 foo
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_1.foo VALUES (1), (2), (3), (4), (5), (6);'
+    sql cdb_testmember_1 'SELECT * FROM cdb_testmember_1.foo;'
+
+    create_table cdb_testmember_2 bar
+    sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3);'
+    sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+}
+
+function tear_down() {
+    log_info "########################### USER TEAR DOWN ###########################"
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
+    sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
+
+    sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
+
+    sql "DROP SCHEMA cartodb CASCADE"
+
+    log_info "########################### TEAR DOWN ###########################"
+    sql 'DROP SCHEMA cdb_testmember_1;'
+    sql 'DROP SCHEMA cdb_testmember_2;'
+
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
+
+    sql 'DROP ROLE cdb_testmember_1;'
+    sql 'DROP ROLE cdb_testmember_2;'
+
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
+
+function run_tests() {
+    local FAILED_TESTS=()
+
+    local TESTS
+    if [[ $# -ge 1 ]]
+    then
+        TESTS="$@"
+    else
+        TESTS=`cat $0 | perl -n -e'/function (test.*)\(\)/ && print "$1\n"'`
+    fi
+    for t in ${TESTS}
+    do
+        echo "####################################################################"
+        echo "#"
+        echo "# Running: ${t}"
+        echo "#"
+        echo "####################################################################"
+        clear_partial_result
+        setup
+        eval ${t}
+        if [[ ${PARTIALOK} -ne 0 ]]
+        then
+            FAILED_TESTS+=(${t})
+        fi
+        tear_down
+    done
+    if [[ ${OK} -ne 0 ]]
+    then
+        echo
+        log_error "The following tests are failing:"
+        printf -- '\t%s\n' "${FAILED_TESTS[@]}"
+    fi
+}
+
+
+#################################################### TESTS GO HERE ####################################################
+
+
+# Tests quota checking taking into account both geom and raster tables
+function test_quota_for_each_user() {
+    # Normal tables add 4096 bytes
+    # Raster tables with overview constraints add 16384 bytes
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+
+    create_raster_table cdb_testmember_1 raster_1
+    create_raster_table cdb_testmember_2 raster_2
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 20480
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    create_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 36864
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 20480
+
+    drop_raster_table cdb_testmember_1 raster_1
+    drop_raster_table cdb_testmember_2 raster_2
+    drop_raster_table cdb_testmember_1 raster_3
+
+    sql cdb_testmember_1 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_1'::TEXT);" should 4096
+    sql cdb_testmember_2 "SELECT cartodb.CDB_UserDataSize('cdb_testmember_2'::TEXT);" should 4096
+}
+
+#################################################### TESTS END HERE ####################################################
+
+run_tests $@
+
+exit ${OK}

test/organization/test.sh

@@ -325,6 +325,27 @@ function test_user_can_read_when_it_has_permission_after_organization_permission
     drop_role_and_schema cdb_testmember_3
 }
 
+function test_cdb_querytables_returns_schema_and_table_name() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
+}
+
+function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
+function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
+    sql "CREATE EXTENSION plpythonu;"
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
+}
+
 #################################################### TESTS END HERE ####################################################