release 0.26.1

Remove default TIS config in Ghost Table function

.travis.yml

@@ -1,46 +1,38 @@
+dist: xenial
 language: c
-dist: precise
 sudo: required
 
-addons:
-  postgresql: 9.5
+env:
+  global:
+    - PGUSER=postgres
+    - PGDATABASE=postgres
+    - PGOPTIONS='-c client_min_messages=NOTICE'
+    - PGPORT=5432
+    - POSTGIS_VERSION="2.5"
+
+  matrix:
+    - POSTGRESQL_VERSION="9.5"
+    - POSTGRESQL_VERSION="10"
+    - POSTGRESQL_VERSION="11"
+
 
 before_install:
-  # Add custom PPAs from cartodb
-  - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
-  - sudo add-apt-repository -y ppa:cartodb/gis
-  - sudo add-apt-repository -y ppa:cartodb/gis-testing
-  - sudo apt-get update
-
-  # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
-  - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
-
-  # Install postgres db and build deps
-  - sudo /etc/init.d/postgresql stop # stop travis default instance
-  - sudo apt-get -y remove --purge postgresql-9.1
-  - sudo apt-get -y remove --purge postgresql-9.2
-  - sudo apt-get -y remove --purge postgresql-9.3
-  - sudo apt-get -y remove --purge postgresql-9.4
-  - sudo apt-get -y remove --purge postgresql-9.5
-  - sudo rm -rf /var/lib/postgresql/
-  - sudo rm -rf /var/log/postgresql/
-  - sudo rm -rf /etc/postgresql/
-  - sudo apt-get -y remove --purge postgis-2.2
-  - sudo apt-get -y autoremove
-
-  - sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3
-  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3
-  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3
-  - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
-  - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
-
-  # configure it to accept local connections from postgres
-  - echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" \
-    | sudo tee /etc/postgresql/9.5/main/pg_hba.conf
-  - sudo /etc/init.d/postgresql restart 9.5
-
+  - sudo service postgresql stop;
+  - sudo apt-get remove postgresql* -y
+  - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
+  - if [[ $POSTGRESQL_VERSION == '9.5' ]]; then sudo apt-get install -y postgresql-contrib-9.5; fi;
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
+  - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
+  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust
+  - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
+  - sudo pip install redis==2.4.9
 script:
   - make
   - sudo make install
-  - PGOPTIONS='-c client_min_messages=NOTICE' PGUSER=postgres make installcheck ||
-    { cat regression.diffs; false; }
+  - make installcheck
+
+after_failure:
+  - pg_lsclusters
+  - cat regression.out
+  - cat regression.diffs

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.20.0
+EXTVERSION = 0.26.1
 
 SED = sed
 AWK = awk
@@ -84,6 +84,18 @@ UPGRADABLE = \
   0.19.1 \
   0.19.2 \
   0.20.0 \
+  0.21.0 \
+  0.22.0 \
+  0.22.1 \
+  0.22.2 \
+  0.23.0 \
+  0.23.1 \
+  0.23.2 \
+  0.24.0 \
+  0.24.1 \
+  0.25.0 \
+  0.26.0 \
+  0.26.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -124,8 +136,10 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	cat cartodb_version.sql >> $@
 ifeq ($(PG_PARALLEL), 0)
 # Remove PARALLEL in aggregates and functions
+	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
 	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' -i $@
+		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
+	mv $(TMPFILE) $@
 endif
 
 $(EXTENSION)--unpackaged--$(EXTVERSION).sql: $(EXTENSION)--$(EXTVERSION).sql util/create_from_unpackaged.sh Makefile

NEWS.md

@@ -1,3 +1,52 @@
+0.26.1 (2019-03-19)
+* Remove default TIS values from Ghost tables functions
+
+0.26.0 (2019-03-11)
+* Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`.
+* Add Ghost tables functions to install triggers and enqueue the linking process
+
+0.25.0 (2019-02-22)
+* Add `CDB_Username` to get the cartodb username from the current PostgreSQL user
+
+0.24.1 (2019-01-02)
+* Drop functions removed in 0.12 (#341)
+* Travis: Test with PostgreSQL 9.5, 10 and 11.
+
+0.24.0 (2018-09-13)
+* Travis: Test with PostgreSQL 9.5 and 10.
+* _cdb_estimated_extent: Fix bug with ST_EstimatedExtent interaction.
+* Improvements in `CDB_JenksBins`.
+  * Now it ignores NULLs.
+  * No longer puts the same value in multiple categories.
+  * Removes all limits related to size.
+  * If not set, the number of iterations done is based now on the size of the array.
+  * Fixed multiple bugs.
+  * The internal function `CDB_JenksBinsIteration` has changed its signature.
+
+0.23.2 (2018-07-19)
+* Fix `CDB_QueryTablesText` with parenthesized queries (#335)
+
+0.23.1 (2018-07-19)
+* Fix `CDB_EstimateRowCount` parallelizability #333
+
+0.23.0 (2018-07-03)
+* Add a new helper function `_CDB_Table_Exists(table_name_with_optional_schema TEXT)` #332
+
+0.22.2 (2018-05-29)
+* Fix: Fix hyphenates usernames in 0.22.1 fix (#331)
+
+0.22.1 (2018-05-29)
+* Fix: Correctly grant permission to all sequences related with table (#330)
+
+0.22.0 (2018-03-22)
+* Fix: allow older ogr2ogr to work in -append mode (#319,#325)
+* Refactors CDB_QuantileBins to rely on PostgreSQL function `percentile_disc` #316
+
+0.21.0 (2018-02-15)
+* Add optional parameter to limit the number of cells in grid-generation functions #322
+* Fix: grant usage on cartodb_id sequence when sharing read write #323
+* Fix: Change sed in-place for tmpfiles 524319
+
 0.20.0 (2017-11-08)
 * Added VOLATILITY and PARALLEL categories to all functions
 

README.md

@@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/
 Dependencies
 ------------
 
- * PostgreSQL 9.3+ (with plpythonu extension and xml support)
+ * PostgreSQL 9.4+ (with plpythonu extension and xml support)
  * [PostGIS extension](http://postgis.net) 
 
 Install

carto-package.json

@@ -0,0 +1,11 @@
+{
+    "name": "carto_postgresql_ext",
+    "current_version": {
+        "requires": {
+            "postgresql": ">=9.5.0",
+            "postgis": ">=2.2.0.0"
+        },
+        "works_with": {
+        }
+    }
+}

scripts-available/CDB_CartodbfyTable.sql

@@ -347,7 +347,7 @@ $$ LANGUAGE PLPGSQL;
 -- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
 --
 --  CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
---    
+--
 --     Main function, calls the following functions, with a little
 --     logic before the table re-write to avoid re-writing if the table
 --     already has all the necessary columns in place.
@@ -468,7 +468,7 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL SAFE;
 
 -- Find out if the table already has a usable primary key
 -- If the table has both a usable key and usable geometry
--- we can no-op on the table copy and just ensure that the 
+-- we can no-op on the table copy and just ensure that the
 -- indexes and triggers are in place
 DROP FUNCTION IF EXISTS _CDB_Has_Usable_Primary_ID(reloid REGCLASS);
 CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Primary_ID(reloid REGCLASS)
@@ -560,7 +560,7 @@ BEGIN
     -- Is there another integer suitable primary key already?
     SELECT a.attname
     INTO rec
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_attribute a ON a.attrelid = c.oid
     JOIN pg_type t ON a.atttypid = t.oid
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
@@ -659,12 +659,12 @@ DECLARE
   rv RECORD;
 
   const RECORD;
-  
+
   has_geom BOOLEAN := false;
   has_mercgeom BOOLEAN := false;
   has_geom_name TEXT;
   has_mercgeom_name TEXT;
-  
+
   -- In case 'the_geom' is a text column
   text_geom_column BOOLEAN := false;
   text_geom_column_name TEXT := '';
@@ -685,12 +685,12 @@ BEGIN
   FOR r1 IN
     SELECT * FROM _cdb_geom_candidate_columns(reloid)
   LOOP
-  
+
     RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('checking column ''%s''', r1.attname);
 
     -- Name collision: right name (the_geom, the_geomwebmercator?)  but wrong type...
     IF r1.typname != 'geometry' AND r1.attname = r1.desired_attname THEN
-    
+
       -- Maybe it's a geometry column hiding in a text column?
       IF r1.typname IN ('text','varchar','char') THEN
 
@@ -710,14 +710,14 @@ BEGIN
           END IF;
         -- Nope, the text in the column can't be converted into geometry
         -- so rename it out of the way
-        EXCEPTION 
+        EXCEPTION
           WHEN others THEN
             IF SQLERRM = 'parse error - invalid geometry' THEN
               text_geom_column := false;
               str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid);
               sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str);
               PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-              RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', 
+              RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %',
                 Format('Text column %s is not convertible to geometry, renamed to %s', r1.attname, str);
             ELSE
               RAISE EXCEPTION 'CDB(_CDB_Has_Usable_Geom) UNEXPECTED ERROR';
@@ -729,7 +729,7 @@ BEGIN
         str := cartodb._CDB_Unique_Column_Identifier(NULL, r1.attname, NULL, reloid);
         sql := Format('ALTER TABLE %s RENAME COLUMN %s TO %I', reloid::text, r1.attname, str);
         PERFORM _CDB_SQL(sql,'_CDB_Has_Usable_Geom');
-        RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', 
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %',
           Format('%s is the wrong type, renamed to %s', r1.attname, str);
       END IF;
 
@@ -749,20 +749,20 @@ BEGIN
           has_mercgeom := true;
           has_mercgeom_name := r1.attname;
         END IF;
-        
+
       -- If it's an unknown SRID, we need to know that too
       ELSIF r1.srid = 0 THEN
-        
+
         -- Unknown SRID, we'll have to fill it in later
         text_geom_column_srid := true;
-        
+
       END IF;
-      
+
     END IF;
-    
+
   END LOOP;
 
-  SELECT 
+  SELECT
     -- If table is perfect (no transforms required), return TRUE!
     has_geom AND has_mercgeom AS has_usable_geoms,
     -- If the geometry column is hiding in a text field, return enough info to deal w/ it.
@@ -772,9 +772,9 @@ BEGIN
     INTO rv;
 
   RAISE DEBUG 'CDB(_CDB_Has_Usable_Geom): %', Format('returning %s', rv);
-  
+
   RETURN rv;
-    
+
 END;
 $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;
 
@@ -782,7 +782,7 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;
 -- Create a copy of the table. Assumes that the "Has usable" functions
 -- have already been run, so that if there is a 'cartodb_id' column, it is
 -- a "good" one, and the same for the geometry columns. If all the required
--- columns are in place already, it no-ops and just renames the table to 
+-- columns are in place already, it no-ops and just renames the table to
 -- the destination if necessary.
 CREATE OR REPLACE FUNCTION _CDB_Rewrite_Table(reloid REGCLASS, destschema TEXT DEFAULT NULL)
 RETURNS BOOLEAN
@@ -791,6 +791,7 @@ DECLARE
 
   relname TEXT;
   relschema TEXT;
+  relseq TEXT;
 
   destoid REGCLASS;
   destname TEXT;
@@ -853,7 +854,7 @@ BEGIN
   -- transformation of the table, we can just ensure proper
   -- indexes are in place and apply a rename
   SELECT *
-  FROM _CDB_Has_Usable_Geom(reloid) 
+  FROM _CDB_Has_Usable_Geom(reloid)
   INTO STRICT gc;
 
   -- If geom is the wrong name, just rename it.
@@ -898,6 +899,21 @@ BEGIN
 
   -- We must rewrite, so here we go...
 
+  -- Our desired PK sequence name
+
+  -- We are going to drop the source table when we're done anyways
+  -- but it's possible the source PK sequence is living in a name we would like to use
+  -- so we check to see if that's the case, and rename it out of the way
+  IF has_usable_primary_key AND has_usable_pk_sequence THEN
+    -- See if the existing sequence is squatting on our preferred name
+    destseq := Format('%s_%s_seq', relname, const.pkey);
+    SELECT pg_catalog.pg_get_serial_sequence(Format('%I.%I', relschema, relname), const.pkey)
+      INTO relseq;
+    -- If it's the name we want, then rename it
+    IF relseq IS NOT NULL AND relseq = Format('%I.%I', destschema, destseq) THEN
+      PERFORM _CDB_SQL(Format('ALTER SEQUENCE %s RENAME TO %I', relseq, Format('tmp_%s', destseq)), '_CDB_Rewrite_Table');
+    END IF;
+  END IF;
 
   -- Put the primary key sequence in the right schema
   -- If the new table is not moving, better ensure the sequence name
@@ -931,16 +947,16 @@ BEGIN
 
     -- Arg, this "geometry" column is actually text!!
     -- OK, we tested back in our geometry column research that it could
-    -- be safely cast to geometry, so let's do that.  
+    -- be safely cast to geometry, so let's do that.
     IF gc.text_geom_column THEN
 
       WITH t AS (
-        SELECT 
+        SELECT
           a.attname,
           CASE WHEN NOT gc.text_geom_column_srid THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start,
           CASE WHEN NOT gc.text_geom_column_srid THEN ',4326)' ELSE '' END AS missing_srid_end
-        FROM pg_class c 
-        JOIN pg_attribute a ON a.attrelid = c.oid 
+        FROM pg_class c
+        JOIN pg_attribute a ON a.attrelid = c.oid
         JOIN pg_type t ON a.atttypid = t.oid
         WHERE c.oid = reloid
         AND t.typname IN ('text','varchar','char')
@@ -954,7 +970,7 @@ BEGIN
             || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end
             || ',4326)::Geometry(GEOMETRY,4326) AS '
             || const.geomcol
-            || ', cartodb.CDB_TransformToWebmercator(' 
+            || ', cartodb.CDB_TransformToWebmercator('
             || t.missing_srid_start || t.attname || '::geometry' || t.missing_srid_end
             || ')::Geometry(GEOMETRY,3857) AS '
             || const.mercgeomcol,
@@ -967,19 +983,19 @@ BEGIN
         -- better be found.
         RAISE EXCEPTION 'CDB(_CDB_Rewrite_Table): Text column % is missing!', gc.text_geom_column_name;
       ELSE
-        sql := sql || geom_transform_sql;  
+        sql := sql || geom_transform_sql;
       END IF;
 
     -- There is at least one true geometry column in here, we'll
-    -- reproject that into the projections we need.  
+    -- reproject that into the projections we need.
     ELSE
 
       -- Find the column we are going to be working with (the first
       -- column with type "geometry")
       SELECT a.attname
       INTO rec
-      FROM pg_class c 
-      JOIN pg_attribute a ON a.attrelid = c.oid 
+      FROM pg_class c
+      JOIN pg_attribute a ON a.attrelid = c.oid
       JOIN pg_type t ON a.atttypid = t.oid
       WHERE c.oid = reloid
       AND t.typname = 'geometry'
@@ -989,16 +1005,16 @@ BEGIN
       LIMIT 1;
 
       -- The SRID could be undeclared at the table level, but still
-      -- exist in the geometries themselves. We first find our geometry 
+      -- exist in the geometries themselves. We first find our geometry
       -- column and read the first SRID off it it, if there is a row
       -- to read.
       IF FOUND THEN
-        EXECUTE Format('SELECT ST_SRID(%s) AS srid FROM %s LIMIT 1', rec.attname, reloid::text) 
+        EXECUTE Format('SELECT ST_SRID(%s) AS srid FROM %s LIMIT 1', rec.attname, reloid::text)
         INTO geom_srid;
       ELSE
         geom_srid := 0;
       END IF;
-  
+
       -- The geometry columns weren't in the right projection,
       -- so we need to find the first decent geometry column
       -- in the table and wrap it in two transforms, one to 4326
@@ -1006,13 +1022,13 @@ BEGIN
       -- ignore it when we build the list of other columns to
       -- add to the output table
       WITH t AS (
-        SELECT 
-          a.attname, 
-          postgis_typmod_type(a.atttypmod) AS geomtype, 
+        SELECT
+          a.attname,
+          postgis_typmod_type(a.atttypmod) AS geomtype,
           CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN 'ST_SetSRID(' ELSE '' END AS missing_srid_start,
           CASE WHEN postgis_typmod_srid(a.atttypmod) = 0 AND srid.srid = 0 THEN ',4326)' ELSE '' END AS missing_srid_end
-        FROM pg_class c 
-        JOIN pg_attribute a ON a.attrelid = c.oid 
+        FROM pg_class c
+        JOIN pg_attribute a ON a.attrelid = c.oid
         JOIN pg_type t ON a.atttypid = t.oid,
         ( SELECT geom_srid AS srid ) AS srid
         WHERE c.oid = reloid
@@ -1022,22 +1038,22 @@ BEGIN
         ORDER BY a.attnum
         LIMIT 1
       )
-      SELECT ', ST_Transform(' 
+      SELECT ', ST_Transform('
             || t.missing_srid_start || t.attname || t.missing_srid_end
             || ',4326)::Geometry(GEOMETRY,4326) AS '
             || const.geomcol
-            || ', cartodb.CDB_TransformToWebmercator(' 
+            || ', cartodb.CDB_TransformToWebmercator('
             || t.missing_srid_start || t.attname || t.missing_srid_end
             || ')::Geometry(GEOMETRY,3857) AS '
             || const.mercgeomcol,
             t.attname
       INTO geom_transform_sql, geom_column_source
       FROM t;
-  
+
       IF NOT FOUND THEN
-        -- If there are no geometry columns, we continue making a 
+        -- If there are no geometry columns, we continue making a
         -- non-spatial table. This is important for folks who want
-        -- their tables to invalidate the SQL API 
+        -- their tables to invalidate the SQL API
         -- cache on update/insert/delete.
         geom_column_source := '';
         sql := sql || ',NULL::geometry(Geometry,4326) AS ' || const.geomcol;
@@ -1057,8 +1073,8 @@ BEGIN
     ',' || array_to_string(array_agg(Format('%I',a.attname)),',') AS column_name_sql,
     Count(*) AS count
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_type t ON a.atttypid = t.oid
   WHERE c.oid = reloid
   AND a.attnum > 0
@@ -1103,11 +1119,11 @@ BEGIN
   sql := Format('ALTER SEQUENCE %s OWNED BY %s.%s', destseq, copyname, const.pkey);
   PERFORM _CDB_SQL(sql,'_CDB_Rewrite_Table');
 
-  
+
   -- We just made a copy, so we can drop the original now
   sql := Format('DROP TABLE %s', reloid::text);
   PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
-  
+
   -- If the table is being created by a SECURITY DEFINER function
   -- make sure the user is set back to the user who is connected
   IF current_user != session_user THEN
@@ -1116,7 +1132,7 @@ BEGIN
     sql := Format('ALTER SEQUENCE IF EXISTS %s OWNER TO %s', destseq, session_user);
     PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
   END IF;
-  
+
   -- If we used a temporary destination table
   -- we can now rename it into place
   IF destschema = relschema THEN
@@ -1155,15 +1171,15 @@ BEGIN
   FROM pg_class c
   WHERE c.oid = reloid;
 
-  -- Is there already a primary key on this table for 
+  -- Is there already a primary key on this table for
   -- a column other than our chosen primary key?
   SELECT ci.relname AS pkey
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
   JOIN pg_class ci ON i.indexrelid = ci.oid
-  WHERE c.oid = reloid 
+  WHERE c.oid = reloid
   AND NOT a.attisdropped
   AND a.attname != const.pkey
   AND i.indisprimary;
@@ -1180,40 +1196,40 @@ BEGIN
   -- Is the default primary key flagged as primary?
   SELECT a.attname
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
   JOIN pg_class ci ON ci.oid = i.indexrelid
-  WHERE attnum > 0 
+  WHERE attnum > 0
   AND c.oid = reloid
   AND a.attname = const.pkey
   AND i.indisprimary
   AND i.indisunique
   AND NOT attisdropped;
-  
+
   -- No primary key? Add one.
   IF NOT FOUND THEN
     sql := Format('ALTER TABLE %s ADD PRIMARY KEY (%s)', reloid::text, const.pkey);
     PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes');
   END IF;
-  
-  -- Add geometry indexes to all "special geometry columns" that 
+
+  -- Add geometry indexes to all "special geometry columns" that
   -- don't have one (either have no index at all, or have a non-GIST index)
-  FOR rec IN 
+  FOR rec IN
     SELECT a.attname, n.nspname
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 
+    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     WHERE NOT attisdropped
     AND a.attname IN (const.geomcol, const.mercgeomcol)
     AND c.oid = reloid
     AND i.indexrelid IS NULL
-    UNION 
+    UNION
     SELECT a.attname, n.nspname
-    FROM pg_class c 
+    FROM pg_class c
     JOIN pg_namespace n ON n.oid = c.relnamespace
-    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0 
+    JOIN pg_attribute a ON a.attrelid = c.oid AND attnum > 0
     JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     JOIN pg_class ci ON ci.oid = i.indexrelid
     JOIN pg_am am ON ci.relam = am.oid
@@ -1225,7 +1241,7 @@ BEGIN
     sql := Format('CREATE INDEX ON %s USING GIST (%s)', reloid::text, rec.attname);
     PERFORM _CDB_SQL(sql, '_CDB_Add_Indexes');
   END LOOP;
-    
+
   RETURN true;
 
 END;
@@ -1236,7 +1252,7 @@ CREATE OR REPLACE FUNCTION CDB_CartodbfyTable(destschema TEXT, reloid REGCLASS)
 RETURNS REGCLASS
 AS $$
 DECLARE
-  
+
   is_raster BOOLEAN;
   relname TEXT;
   relschema TEXT;
@@ -1245,13 +1261,13 @@ DECLARE
   destname TEXT;
 
   rec RECORD;
-  
+
 BEGIN
 
   -- Save the raw schema/table names for later
   SELECT n.nspname, c.relname, c.relname
   INTO STRICT relschema, relname, destname
-  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid 
+  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
   WHERE c.oid = reloid;
 
   PERFORM cartodb._CDB_check_prerequisites(destschema, reloid);
@@ -1259,7 +1275,7 @@ BEGIN
   -- Check destination schema exists
   -- Throws an exception of there is no matching schema
   IF destschema IS NOT NULL THEN
-  
+
     SELECT n.nspname
     INTO rec FROM pg_namespace n WHERE n.nspname = destschema;
     IF NOT FOUND THEN
@@ -1282,7 +1298,7 @@ BEGIN
     PERFORM cartodb._CDB_create_raster_triggers(destschema, reloid);
 
   ELSE
-    
+
     -- Rewrite (or rename) the table to the new location
     PERFORM _CDB_Rewrite_Table(reloid, destschema);
 
@@ -1291,7 +1307,7 @@ BEGIN
 
     -- Add indexes to the destination table, as necessary
     PERFORM _CDB_Add_Indexes(destoid);
-  
+
     -- Add triggers to the destination table, as necessary
     PERFORM _CDB_create_triggers(destschema, destoid);
 

scripts-available/CDB_DDLTriggers.sql

@@ -0,0 +1,14 @@
+--
+-- Legacy file
+-- Introduced again to make sure that updates do not leave dangling functions
+--
+
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_create_table();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_drop_table();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_alter_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_drop_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_add_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_disable_ddl_hooks();
+DROP FUNCTION IF EXISTS cartodb.cdb_enable_ddl_hooks();
+
+

scripts-available/CDB_EstimateRowCount.sql

@@ -12,7 +12,7 @@ BEGIN
     EXECUTE Format('ANALYZE %s;', reloid);
   END IF;
 END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL RESTRICTED SECURITY DEFINER;
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE SECURITY DEFINER;
 
 -- Return a row count estimate of the result of a query using statistics
 CREATE OR REPLACE FUNCTION CDB_EstimateRowCount(query text)
@@ -28,4 +28,4 @@ BEGIN
   EXECUTE 'EXPLAIN (FORMAT JSON) ' || query INTO STRICT plan;
   RETURN plan->0->'Plan'->'Plan Rows';
 END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL RESTRICTED;
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE;

scripts-available/CDB_GhostTables.sql

@@ -0,0 +1,123 @@
+-- Enqueues a job to run Ghost tables linking process for the provided username
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text) 
+RETURNS void
+AS $$
+  if not username:
+    return
+
+  if 'json' not in GD:
+    import json
+    GD['json'] = json
+  else:
+    json = GD['json']    
+
+  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
+  if not tis_config:
+    plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
+    return
+
+  tis_config_dict = json.loads(tis_config)
+  tis_host = tis_config_dict.get('host')
+  tis_port = tis_config_dict.get('port')
+  tis_timeout = tis_config_dict.get('timeout', 5)
+  tis_retry = tis_config_dict.get('retry', 5)
+      
+  client = GD.get('invalidation', None)
+
+  while True:
+
+    if not client:
+        try:
+          import redis
+          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
+          GD['invalidation'] = client
+        except Exception as err:
+          error = "client_error - %s" % str(err)
+          # NOTE: no retries on connection error
+          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' +  str(err))
+          break
+
+    try:
+      client.execute_command('DBSCH', db_name, username, event_name)
+      break
+    except Exception as err:
+      error = "request_error - %s" % str(err)
+      client = GD['invalidation'] = None # force reconnect
+      if not tis_retry:
+        plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' +  str(err))
+        break
+      tis_retry -= 1 # try reconnecting
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+
+-- Enqueues a job to run Ghost tables linking process for the current user
+CREATE OR REPLACE FUNCTION cartodb.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
+RETURNS void
+AS $$
+  DECLARE
+    username TEXT;
+    db_name TEXT;
+  BEGIN
+    EXECUTE 'SELECT cartodb.CDB_Username();' INTO username;
+    EXECUTE 'SELECT current_database();' INTO db_name;
+
+    PERFORM cartodb._CDB_LinkGhostTables(username, db_name, event_name);
+    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Trigger function to call CDB_LinkGhostTables()
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTablesTrigger()
+RETURNS trigger
+AS $$
+  DECLARE
+    ddl_tag TEXT;
+  BEGIN
+    EXECUTE 'DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current() RETURNING tag;' INTO ddl_tag;
+    PERFORM cartodb.CDB_LinkGhostTables(ddl_tag);
+    RETURN NULL;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Event trigger to save the current transaction in cartodb.cdb_ddl_execution
+CREATE OR REPLACE FUNCTION cartodb.CDB_SaveDDLTransaction()
+RETURNS event_trigger
+AS $$
+  BEGIN
+    INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Creates the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION cartodb.CDB_EnableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+
+    -- Table to store the transaction id from DDL events to avoid multiple executions
+    CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text);
+
+    CREATE CONSTRAINT TRIGGER check_ddl_update
+    AFTER INSERT ON cartodb.cdb_ddl_execution
+    INITIALLY DEFERRED
+    FOR EACH ROW
+    EXECUTE PROCEDURE cartodb._CDB_LinkGhostTablesTrigger();
+
+    CREATE EVENT TRIGGER link_ghost_tables
+    ON ddl_command_end
+    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW')
+    EXECUTE PROCEDURE cartodb.CDB_SaveDDLTransaction();
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+-- Drops the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION cartodb.CDB_DisableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+    DROP TABLE IF EXISTS cartodb.cdb_ddl_execution;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_GreatCircle.sql

@@ -16,7 +16,7 @@ BEGIN
 
   IF ST_XMax(line) - ST_XMin(line) > 180 THEN
     line = ST_Difference(
-      ST_Shift_Longitude(line),
+      ST_ShiftLongitude(line),
 			ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001)
 		);
   END IF;

scripts-available/CDB_Helper.sql

@@ -158,3 +158,20 @@ BEGIN
   RETURN left(string, (i - 1));
 END;
 $$ LANGUAGE 'plpgsql' IMMUTABLE PARALLEL SAFE;
+
+
+-- Checks if a given text representing a qualified or unqualified table name (relation)
+-- actually exists in the database. It is meant to be used as a guard for other function/queries.
+CREATE OR REPLACE FUNCTION cartodb._CDB_Table_Exists(table_name_with_optional_schema TEXT)
+RETURNS bool
+AS $$
+DECLARE
+    table_exists bool := false;
+BEGIN
+    table_exists := EXISTS(SELECT * FROM pg_class WHERE table_name_with_optional_schema::regclass::oid = oid AND relkind = 'r');
+    RETURN table_exists;
+EXCEPTION
+    WHEN invalid_schema_name OR undefined_table THEN
+        RETURN false;
+END;
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Hexagon.sql

@@ -10,6 +10,10 @@ AS $$
     WHERE path[1] % 2 != 0
 $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE;
 
+
+-- In older versions of the extension, CDB_HexagonGrid had a different signature
+DROP FUNCTION IF EXISTS cartodb.CDB_HexagonGrid(GEOMETRY, FLOAT8, GEOMETRY);
+
 --
 -- Fill given extent with an hexagonal coverage
 --
@@ -25,10 +29,13 @@ $$ LANGUAGE 'sql' IMMUTABLE STRICT PARALLEL SAFE;
 --               If omitted the origin will be 0,0.
 --               The parameter is checked for having the same SRID
 --               as the extent.
---  
 --
+-- @param maxcells Optional maximum number of grid cells to generate;
+--                 if the grid requires more cells to cover the extent
+--                 and exception will occur.
+----
 -- DROP FUNCTION IF EXISTS CDB_HexagonGrid(ext GEOMETRY, side FLOAT8);
-CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL)
+CREATE OR REPLACE FUNCTION CDB_HexagonGrid(ext GEOMETRY, side FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512)
 RETURNS SETOF GEOMETRY
 AS $$
 DECLARE
@@ -107,6 +114,12 @@ BEGIN
     vstartary := ARRAY[ vstart - (vstep/2.0), vstart ];
   END IF;
 
+  If maxcells IS NOT NULL AND maxcells > 0 THEN
+    IF CEIL((CEIL((vend-vstart)/(vstep/2.0)) * CEIL((hend-hstart)/(hstep*2.0/3.0)))/3.0)::integer > maxcells THEN
+      RAISE EXCEPTION 'The requested grid is too big to be rendered';
+    END IF;
+  END IF;
+
   vstartidx := abs(hskip)%2;
 
   RAISE DEBUG 'vstartary: % : %', vstartary[1], vstartary[2];

scripts-available/CDB_JenksBins.sql

@@ -10,212 +10,337 @@
 --
 -- @param invert Optional wheter to return the top of each bin (default)
 --               or the bottom. BOOLEAN, default=FALSE.
---  
+--
 --
 
+CREATE OR REPLACE FUNCTION CDB_JenksBins(in_array NUMERIC[], breaks INT, iterations INT DEFAULT 0, invert BOOLEAN DEFAULT FALSE)
+RETURNS NUMERIC[] as
+$$
+DECLARE
+    in_matrix NUMERIC[][];
+    in_unique_count BIGINT;
 
-CREATE OR REPLACE FUNCTION CDB_JenksBins ( in_array NUMERIC[], breaks INT, iterations INT DEFAULT 5, invert BOOLEAN DEFAULT FALSE) RETURNS NUMERIC[] as $$ 
-DECLARE 
-    element_count INT4; 
+    shuffles INT;
     arr_mean NUMERIC;
+    sdam NUMERIC;
+
+    i INT;
     bot INT;
     top INT;
+
     tops INT[];
     classes INT[][];
-    i INT := 1; j INT := 1; 
+    j INT := 1;
     curr_result NUMERIC[];
     best_result NUMERIC[];
     seedtarget TEXT;
-    quant NUMERIC[];
-    shuffles INT;
+
 BEGIN
-    -- get the total size of our row
-    element_count := array_length(in_array, 1); --array_upper(in_array, 1) - array_lower(in_array, 1); 
-    -- ensure the ordering of in_array
-    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e) x;
-    -- stop if no rows 
-    IF element_count IS NULL THEN  
-        RETURN NULL; 
-    END IF; 
-    -- stop if our breaks are more than our input array size
-    IF element_count < breaks THEN  
-        RETURN in_array; 
-    END IF; 
+    -- We clean the input array (remove NULLs) and create 2 arrays
+    -- [1] contains the unique values in in_array
+    -- [2] contains the number of appearances of those unique values
+    SELECT ARRAY[array_agg(value), array_agg(count)] FROM
+    (
+        SELECT value, count(1)::numeric as count
+        FROM  unnest(in_array) AS value
+        WHERE value is NOT NULL
+        GROUP BY value
+        ORDER BY value
+    ) __clean_array_q INTO in_matrix;
 
-    shuffles := LEAST(GREATEST(floor(2500000.0/(element_count::float*iterations::float)), 1), 750)::int;
-    -- get our mean value
-    SELECT avg(v) INTO arr_mean FROM (  SELECT unnest(in_array) as v ) x; 
+    -- Get the number of unique values
+    in_unique_count := array_length(in_matrix[1:1], 2);
 
-    -- assume best is actually Quantile
-    SELECT CDB_QuantileBins(in_array, breaks) INTO quant;
-
-    -- if data is very very large, just return quant and be done
-    IF element_count > 5000000 THEN
-        RETURN quant;
+    IF in_unique_count IS NULL THEN
+        RETURN NULL;
     END IF;
 
-    -- change quant into bottom, top markers
-    LOOP 
-        IF i = 1 THEN 
-            bot = 1;
-        ELSE 
-            -- use last top to find this bot
-            bot = top+1;
-        END IF;
-        IF i = breaks THEN
-            top = element_count;
+    IF in_unique_count <= breaks THEN
+        -- There isn't enough distinct values for the requested breaks
+        RETURN ARRAY(Select unnest(in_matrix[1:1])) _a;
+    END IF;
+
+    -- If not declated explicitly we iterate based on the length of the array
+    IF iterations < 1 THEN
+        -- This is based on a 'looks fine' heuristic
+        iterations := log(in_unique_count)::integer + 1;
+    END IF;
+
+    -- We set the number of shuffles per iteration as the number of unique values but
+    -- this is just another 'looks fine' heuristic
+    shuffles := in_unique_count;
+
+    -- Get the mean value of the whole vector (already ignores NULLs)
+    SELECT avg(v) INTO arr_mean FROM ( SELECT unnest(in_array) as v ) x;
+
+    -- Calculate the sum of squared deviations from the array mean (SDAM).
+    SELECT sum(((arr_mean - v)^2) * w) INTO sdam FROM (
+        SELECT unnest(in_matrix[1:1]) as v, unnest(in_matrix[2:2]) as w
+        ) x;
+
+    -- To start, we create ranges with approximately the same amount of different values
+    top := 0;
+    i := 1;
+    LOOP
+        bot := top + 1;
+        top := ROUND(i * in_unique_count::numeric / breaks::NUMERIC);
+
+        IF i = 1 THEN
+            classes = ARRAY[ARRAY[bot,top]];
         ELSE
-            SELECT count(*) INTO top FROM ( SELECT unnest(in_array) as v) x WHERE v <= quant[i];
-        END IF;
-        IF i = 1 THEN 
-            classes = ARRAY[ARRAY[bot,top]]; 
-        ELSE 
-            classes = ARRAY_CAT(classes,ARRAY[bot,top]); 
+            classes = ARRAY_CAT(classes, ARRAY[bot,top]);
         END IF;
+
+        i := i + 1;
         IF i > breaks THEN EXIT; END IF;
-        i = i+1;
     END LOOP;
 
-    best_result = CDB_JenksBinsIteration( in_array, breaks, classes, invert, element_count, arr_mean, shuffles);
+    best_result = CDB_JenksBinsIteration(in_matrix, breaks, classes, invert, sdam, shuffles);
 
     --set the seed so we can ensure the same results
     SELECT setseed(0.4567) INTO seedtarget;
     --loop through random starting positions
     LOOP
-        IF j > iterations-1 THEN  EXIT;  END IF;  
+        IF j > iterations-1 THEN  EXIT;  END IF;
         i = 1;
-        tops = ARRAY[element_count];
+        tops = ARRAY[in_unique_count];
         LOOP
-            IF i = breaks THEN  EXIT;  END IF;  
-            SELECT array_agg(distinct e) INTO tops FROM (SELECT unnest(array_cat(tops, ARRAY[floor(random()*element_count::float)::int])) as e ORDER BY e) x WHERE e != 1;
-            i = array_length(tops, 1); 
-        END LOOP; 
+            IF i = breaks THEN  EXIT;  END IF;
+            SELECT array_agg(distinct e) INTO tops FROM (
+                SELECT unnest(array_cat(tops, ARRAY[trunc(random() * in_unique_count::float8)::int + 1])) as e ORDER BY e
+                ) x;
+            i = array_length(tops, 1);
+        END LOOP;
+        top := 0;
         i = 1;
-        LOOP  
-            IF i > breaks THEN  EXIT;  END IF;  
-            IF i = 1 THEN
-                bot = 1;
-            ELSE
-                bot = top+1;
-            END IF;
+        LOOP
+            bot := top + 1;
             top = tops[i];
-            IF i = 1 THEN 
-                classes = ARRAY[ARRAY[bot,top]]; 
-            ELSE 
-                classes = ARRAY_CAT(classes,ARRAY[bot,top]); 
+            IF i = 1 THEN
+                classes = ARRAY[ARRAY[bot,top]];
+            ELSE
+                classes = ARRAY_CAT(classes, ARRAY[bot,top]);
             END IF;
-            i := i+1; 
-        END LOOP; 
-        curr_result = CDB_JenksBinsIteration( in_array, breaks, classes, invert, element_count, arr_mean, shuffles);
+
+            i := i+1;
+            IF i > breaks THEN EXIT; END IF;
+        END LOOP;
+
+        curr_result = CDB_JenksBinsIteration(in_matrix, breaks, classes, invert, sdam, shuffles);
 
         IF curr_result[1] > best_result[1] THEN
             best_result = curr_result;
-            j = j-1; -- if we found a better result, add one more search
         END IF;
+
         j = j+1;
     END LOOP;
 
     RETURN (best_result)[2:array_upper(best_result, 1)];
 END;
-$$ language plpgsql VOLATILE PARALLEL RESTRICTED;
-
+$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL RESTRICTED;
 
 
 --
 -- Perform a single iteration of the Jenks classification
 --
+-- Returns an array with:
+-- - First element: gvf
+-- - Second to 2+n: Category limits
+DROP FUNCTION IF EXISTS CDB_JenksBinsIteration ( in_matrix NUMERIC[], breaks INT, classes INT[], invert BOOLEAN, element_count INT4, arr_mean NUMERIC, max_search INT); -- Old signature
+
+CREATE OR REPLACE FUNCTION CDB_JenksBinsIteration ( in_matrix NUMERIC[], breaks INT, classes INT[], invert BOOLEAN, sdam NUMERIC, max_search INT DEFAULT 50) RETURNS NUMERIC[] as $$
+DECLARE
+    i INT;
+    iterations INT = 0;
 
-CREATE OR REPLACE FUNCTION CDB_JenksBinsIteration ( in_array NUMERIC[], breaks INT, classes INT[][], invert BOOLEAN, element_count INT4, arr_mean NUMERIC, max_search INT DEFAULT 50) RETURNS NUMERIC[] as $$ 
-DECLARE 
-    tmp_val numeric; 
-    new_classes int[][];
-    tmp_class int[];
-    i INT := 1; 
-    j INT := 1; 
     side INT := 2;
-    sdam numeric; 
-    gvf numeric := 0.0; 
-    new_gvf numeric; 
-    arr_gvf numeric[]; 
-    class_avg numeric; 
-    class_max_i INT; 
-    class_min_i INT; 
-    class_max numeric; 
-    class_min numeric; 
-    reply numeric[]; 
-BEGIN 
 
-    -- Calculate the sum of squared deviations from the array mean (SDAM).
-    SELECT sum((arr_mean - e)^2) INTO sdam FROM (  SELECT unnest(in_array) as e ) x; 
-    --Identify the breaks for the lowest GVF
-    LOOP   
-        i = 1; 
-        LOOP 
-            -- get our mean
-            SELECT avg(e) INTO class_avg FROM ( SELECT unnest(in_array[classes[i][1]:classes[i][2]]) as e) x;  
-            -- find the deviation
-            SELECT sum((class_avg-e)^2) INTO tmp_val FROM (   SELECT unnest(in_array[classes[i][1]:classes[i][2]]) as e  ) x;  
-            IF i = 1 THEN 
-                arr_gvf = ARRAY[tmp_val]; 
-                -- init our min/max map for later
-                class_max = arr_gvf[i];  
-                class_min = arr_gvf[i];  
-                class_min_i = 1;   
-                class_max_i = 1;  
-            ELSE 
-                arr_gvf = array_append(arr_gvf, tmp_val); 
-            END IF;
-            i := i+1;  
-            IF i > breaks THEN EXIT; END IF;  
-        END LOOP;  
-        -- calculate our new GVF
-        SELECT sdam-sum(e) INTO new_gvf FROM (  SELECT unnest(arr_gvf) as e  ) x;  
-        -- if no improvement was made, exit
-        IF new_gvf < gvf THEN EXIT; END IF; 
-        gvf = new_gvf;  
-        IF j > max_search THEN EXIT; END IF; 
-        j = j+1;
-        i = 1;  
-        LOOP  
-            --establish directionality (uppward through classes or downward)
-            IF arr_gvf[i] < class_min THEN   
-                class_min = arr_gvf[i];   
-                class_min_i = i;  
-            END IF;  
-            IF arr_gvf[i] > class_max THEN   
-                class_max = arr_gvf[i];   
-                class_max_i = i;  
-            END IF;  
-            i := i+1;  
-            IF i > breaks THEN EXIT; END IF;  
-        END LOOP;  
-        IF class_max_i > class_min_i THEN
-            class_min_i = class_max_i - 1;
+    gvf numeric := 0.0;
+    new_gvf numeric;
+    arr_gvf numeric[];
+    arr_avg numeric[];
+    class_avg numeric;
+    class_dev numeric;
+
+    class_max_i INT = 0;
+    class_min_i INT = 0;
+    dev_max numeric;
+    dev_min numeric;
+
+    best_classes INT[] = classes;
+    best_gvf numeric[];
+    best_avg numeric[];
+    move_elements INT = 1;
+
+    reply numeric[];
+
+BEGIN
+
+    -- We fill the arrays with the initial values
+    i = 0;
+    LOOP
+        IF i = breaks THEN EXIT; END IF;
+        i = i + 1;
+
+        -- Get class mean
+        SELECT (sum(v * w) / sum(w)) INTO class_avg FROM (
+            SELECT unnest(in_matrix[1:1][classes[i][1]:classes[i][2]]) as v,
+                    unnest(in_matrix[2:2][classes[i][1]:classes[i][2]]) as w
+            ) x;
+
+        -- Get class deviation
+        SELECT sum((class_avg - v)^2 * w) INTO class_dev FROM (
+            SELECT unnest(in_matrix[1:1][classes[i][1]:classes[i][2]]) as v,
+                    unnest(in_matrix[2:2][classes[i][1]:classes[i][2]]) as w
+            ) x;
+
+
+        IF i = 1 THEN
+            arr_avg = ARRAY[class_avg];
+            arr_gvf = ARRAY[class_dev];
         ELSE
-            class_min_i = class_max_i + 1;
+            arr_avg = array_append(arr_avg, class_avg);
+            arr_gvf = array_append(arr_gvf, class_dev);
         END IF;
-            --Move from higher class to a lower gid order
-            IF class_max_i > class_min_i THEN
-                classes[class_max_i][1] = classes[class_max_i][1] + 1;
-                classes[class_min_i][2] = classes[class_min_i][2] + 1;
-            ELSE -- Move from lower class UP into a higher class by gid
-                classes[class_max_i][2] = classes[class_max_i][2] - 1;
-                classes[class_min_i][1] = classes[class_min_i][1] - 1;
+    END LOOP;
+
+    -- We copy the values to avoid recalculation when a failure happens
+    best_avg = arr_avg;
+    best_gvf = arr_gvf;
+
+    iterations = 0;
+    LOOP
+        IF iterations = max_search THEN EXIT; END IF;
+        iterations = iterations + 1;
+
+        -- calculate our new GVF
+        SELECT sdam - sum(e) INTO new_gvf FROM ( SELECT unnest(arr_gvf) as e ) x;
+
+        -- Check if any improvement was made
+        IF new_gvf <= gvf THEN
+            -- If we were moving too many elements, go back and move less
+            IF move_elements <= 2 OR class_max_i = class_min_i THEN
+                EXIT;
             END IF;
-    END LOOP; 
+
+            move_elements = GREATEST(move_elements / 8, 1);
+
+            -- Rollback from saved statuses
+            classes = best_classes;
+            new_gvf = gvf;
+
+            i = class_min_i;
+            LOOP
+                arr_avg[i] = best_avg[i];
+                arr_gvf[i] = best_gvf[i];
+
+                IF i = class_max_i THEN EXIT; END IF;
+                i = i + 1;
+            END LOOP;
+        END IF;
+
+        -- We search for the classes with the min and max deviation
+        i = 1;
+        class_min_i = 1;
+        class_max_i = 1;
+        dev_max = arr_gvf[1];
+        dev_min = arr_gvf[1];
+        LOOP
+            IF i = breaks THEN EXIT; END IF;
+            i = i + 1;
+
+            IF arr_gvf[i] < dev_min THEN
+                dev_min = arr_gvf[i];
+                class_min_i = i;
+            ELSE
+                IF arr_gvf[i] > dev_max THEN
+                    dev_max = arr_gvf[i];
+                    class_max_i = i;
+                END IF;
+            END IF;
+        END LOOP;
+
+
+        -- Save best values for comparison and output
+        gvf = new_gvf;
+        best_classes = classes;
+
+        -- Limit the moved elements as to not remove everything from class_max_i
+        move_elements = LEAST(move_elements, classes[class_max_i][2] - classes[class_max_i][1]);
+
+        -- Move `move_elements` from class_max_i to class_min_i
+        IF class_min_i < class_max_i THEN
+            i := class_min_i;
+            LOOP
+                IF i = class_max_i THEN EXIT; END IF;
+                classes[i][2] = classes[i][2] + move_elements;
+                i := i + 1;
+            END LOOP;
+
+            i := class_max_i;
+            LOOP
+                IF i = class_min_i THEN EXIT; END IF;
+                classes[i][1] = classes[i][1] + move_elements;
+                i := i - 1;
+            END LOOP;
+        ELSE
+            i := class_min_i;
+            LOOP
+                IF i = class_max_i THEN EXIT; END IF;
+                classes[i][1] = classes[i][1] - move_elements;
+                i := i - 1;
+            END LOOP;
+
+            i := class_max_i;
+            LOOP
+                IF i = class_min_i THEN EXIT; END IF;
+                classes[i][2] = classes[i][2] - move_elements;
+                i := i + 1;
+            END LOOP;
+        END IF;
+
+        -- Recalculate avg and deviation ONLY for the affected classes
+        i = LEAST(class_min_i, class_max_i);
+        class_max_i = GREATEST(class_min_i, class_max_i);
+        class_min_i = i;
+        LOOP
+            SELECT (sum(v * w) / sum(w)) INTO class_avg FROM (
+                SELECT unnest(in_matrix[1:1][classes[i][1]:classes[i][2]]) as v,
+                        unnest(in_matrix[2:2][classes[i][1]:classes[i][2]]) as w
+                ) x;
+
+            SELECT sum((class_avg - v)^2 * w) INTO class_dev FROM (
+                SELECT unnest(in_matrix[1:1][classes[i][1]:classes[i][2]]) as v,
+                        unnest(in_matrix[2:2][classes[i][1]:classes[i][2]]) as w
+                ) x;
+
+            -- Save status (in case it's needed for rollback) and store the new one
+            best_avg[i] = arr_avg[i];
+            arr_avg[i] = class_avg;
+
+            best_gvf[i] = arr_gvf[i];
+            arr_gvf[i] = class_dev;
+
+            IF i = class_max_i THEN EXIT; END IF;
+            i = i + 1;
+        END LOOP;
+
+        move_elements = move_elements * 2;
+
+    END LOOP;
 
     i = 1;
-    LOOP  
+    LOOP
         IF invert = TRUE THEN
             side = 1; --default returns bottom side of breaks, invert returns top side
         END IF;
-        reply = array_append(reply, in_array[classes[i][side]]);  
-        i = i+1;  
-        IF i > breaks THEN  EXIT; END IF; 
-    END LOOP; 
-    
-    RETURN array_prepend(gvf, reply); 
+        reply = array_append(reply, unnest(in_matrix[1:1][best_classes[i][side]:best_classes[i][side]]));
+        i = i+1;
+        IF i > breaks THEN  EXIT; END IF;
+    END LOOP;
 
-END; 
-$$ language plpgsql IMMUTABLE PARALLEL SAFE;
+    reply = array_prepend(gvf, reply);
+    RETURN reply;
 
+END;
+$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;

scripts-available/CDB_Organizations.sql

@@ -104,13 +104,38 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Organization_Get_Table_Sequences(from_schema text, table_name text)
+    RETURNS SETOF TEXT
+AS $$
+BEGIN
+    RETURN QUERY EXECUTE 'SELECT
+        quote_ident(n.nspname) || ''.'' || quote_ident(c.relname)
+    FROM
+        pg_depend d
+        JOIN pg_class c ON d.objid = c.oid
+        JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE
+        d.refobjsubid > 0 AND
+        d.classid = ''pg_class''::regclass AND
+        c.relkind = ''S''::"char" AND
+        d.refobjid = (''' || quote_ident(from_schema) || '.' || quote_ident(table_name) ||''')::regclass';
+END
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
+
 CREATE OR REPLACE
 FUNCTION cartodb.CDB_Organization_Add_Table_Read_Write_Permission(from_schema text, table_name text, to_role_name text)
     RETURNS void
 AS $$
+DECLARE
+    sequence_name TEXT;
 BEGIN
     EXECUTE 'GRANT USAGE ON SCHEMA "' || from_schema || '" TO "' || to_role_name || '"';
     EXECUTE 'GRANT SELECT, INSERT, UPDATE, DELETE ON "' || from_schema || '"."' || table_name || '" TO "' || to_role_name || '"';
+
+    FOR sequence_name IN SELECT * FROM cartodb._CDB_Organization_Get_Table_Sequences(from_schema, table_name) LOOP
+        EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE ' || sequence_name || ' TO "' || to_role_name || '"';
+    END LOOP;
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 

scripts-available/CDB_Overviews.sql

@@ -93,11 +93,8 @@ AS $$
       table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
     );
 
-    BEGIN
-      EXECUTE ext_query INTO ext;
-    EXCEPTION
-        -- This is the typical ERROR: stats for "mytable" do not exist
-        WHEN internal_error THEN
+    EXECUTE ext_query INTO ext;
+    IF ext IS NULL THEN
           -- Get stats and execute again
           EXECUTE format('ANALYZE %1$s', reloid);
 
@@ -107,7 +104,7 @@ AS $$
           END IF;
 
           EXECUTE ext_query INTO ext;
-    END;
+    END IF;
 
     RETURN ext;
   END;

scripts-available/CDB_QuantileBins.sql

@@ -5,39 +5,14 @@
 --            bins based on the Quantile method.
 --
 -- @param breaks The number of bins you want to find.
---  
 --
-CREATE OR REPLACE FUNCTION CDB_QuantileBins ( in_array NUMERIC[], breaks INT) RETURNS NUMERIC[] as $$ 
-DECLARE 
-    element_count INT4; 
-    break_size numeric;
-    tmp_val numeric; 
-    i INT := 1; 
-    reply numeric[]; 
-BEGIN 
-    -- sort our values
-    SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e ASC) x;
-    -- get the total size of our data
-    element_count := array_length(in_array, 1); 
-    break_size :=  element_count::numeric / breaks;
-    -- slice our bread
-    LOOP  
-        IF i < breaks THEN
-            IF break_size * i % 1 > 0 THEN
-                SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET ceil(break_size * i) - 1) x;
-            ELSE
-                SELECT avg(e) INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 2 OFFSET ceil(break_size * i) - 1 ) x;
-            END IF;
-        ELSIF i = breaks THEN
-            -- select the last value
-            SELECT max(e) INTO tmp_val FROM ( SELECT unnest(in_array) e ) x;
-        ELSE
-            EXIT;
-        END IF;
-
-        reply = array_append(reply, tmp_val);
-        i := i+1;
-    END LOOP;
-    RETURN reply;
-END; 
-$$ language plpgsql IMMUTABLE STRICT PARALLEL SAFE;
+--
+CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int)
+RETURNS numeric[]
+AS $$
+  SELECT
+    percentile_disc(Array(SELECT generate_series(1, breaks) / breaks::numeric))
+    WITHIN GROUP (ORDER BY x ASC) AS p
+  FROM
+    unnest(in_array) AS x;
+$$ language SQL IMMUTABLE STRICT PARALLEL SAFE;

scripts-available/CDB_QueryTables.sql

@@ -11,19 +11,16 @@ DECLARE
   rec RECORD;
   rec2 RECORD;
 BEGIN
-  
+
   tables := '{}';
 
   FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
-
-    IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
-        --RAISE WARNING 'Skipping %', rec.q;
-        CONTINUE;
-    END IF;
-
     BEGIN
       EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
-    EXCEPTION WHEN others THEN
+    EXCEPTION WHEN syntax_error THEN
+      -- We can get a syntax error if the user tries to EXPLAIN a DDL
+      CONTINUE;
+    WHEN others THEN
       -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
       --       the affected table ?
       RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;

scripts-available/CDB_RectangleGrid.sql

@@ -1,3 +1,6 @@
+-- In older versions of the extension, CDB_RectangleGrid had a different signature
+DROP FUNCTION IF EXISTS cartodb.CDB_RectangleGrid(GEOMETRY, FLOAT8, FLOAT8, GEOMETRY);
+
 --
 -- Fill given extent with a rectangular coverage
 --
@@ -14,9 +17,12 @@
 --               If omitted the origin will be 0,0.
 --               The parameter is checked for having the same SRID
 --               as the extent.
---  
 --
-CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL)
+-- @param maxcells Optional maximum number of grid cells to generate;
+--                 if the grid requires more cells to cover the extent
+--                 and exception will occur.
+--
+CREATE OR REPLACE FUNCTION CDB_RectangleGrid(ext GEOMETRY, width FLOAT8, height FLOAT8, origin GEOMETRY DEFAULT NULL, maxcells INTEGER DEFAULT 512*512)
 RETURNS SETOF GEOMETRY
 AS $$
 DECLARE
@@ -79,6 +85,12 @@ BEGIN
   --RAISE DEBUG 'hend: %', hend;
   --RAISE DEBUG 'vend: %', vend;
 
+  If maxcells IS NOT NULL AND maxcells > 0 THEN
+    IF ((hend - hstart)/hstep * (vend - vstart)/vstep)::integer > maxcells THEN
+        RAISE EXCEPTION 'The requested grid is too big to be rendered';
+    END IF;
+  END IF;
+
   x := hstart;
   WHILE x < hend LOOP -- over X
     y := vstart;

scripts-available/CDB_TableMetadata.sql

@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS
 
 CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS
        SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at
-       FROM public.CDB_TableMetadata, pg_catalog.pg_class c
+       FROM public.CDB_TableMetadata m JOIN pg_catalog.pg_class c ON m.tabname::oid = c.oid
        LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid;
 
 -- No one can see this

scripts-available/CDB_TransformToWebmercator.sql

@@ -31,13 +31,17 @@ BEGIN
 
   -- Then we transform to WGS84 latlon, which is
   -- where we have known coordinates for the clipping
-  -- 
+  --
   latlon_input := ST_Transform(geom, 4326);
 
   -- Don't bother clipping if the geometry boundary doesn't
   -- go outside the valid extent.
   IF latlon_input @ valid_extent THEN
-    RETURN ST_Transform(latlon_input, 3857);
+    BEGIN
+      RETURN ST_Transform(latlon_input, 3857);
+    EXCEPTION WHEN OTHERS THEN
+      RETURN NULL;
+    END;
   END IF;
 
   -- Since we're going to use ST_Intersection on input
@@ -65,7 +69,7 @@ BEGIN
   -- We transform to web mercator
   to_webmercator := ST_Transform(clipped_input, 3857);
 
-  -- Finally we convert EMPTY to NULL 
+  -- Finally we convert EMPTY to NULL
   -- See https://github.com/Vizzuality/cartodb/issues/706
   -- And retain "multi" status
   ret := CASE WHEN ST_IsEmpty(to_webmercator) THEN NULL::geometry

scripts-available/CDB_Username.sql

@@ -0,0 +1,6 @@
+-- Returns the cartodb username of the current PostgreSQL session
+CREATE OR REPLACE FUNCTION cartodb.CDB_Username()
+RETURNS text
+AS $$
+  SELECT cartodb.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
+$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;

scripts-enabled/290-CDB_GhostTables.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_GhostTables.sql

scripts-enabled/900-CDB_DDLTriggers.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DDLTriggers.sql

scripts-enabled/920-CDB_Username.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Username.sql

test/CDB_CartodbfyTableTest.sql

@@ -385,6 +385,11 @@ BEGIN;
     $$;
 ROLLBACK;
 
+-- Long table name could cause possible sequence rename collision #325
+CREATE TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8" ( cartodb_id serial primary key );
+SELECT CDB_CartodbfyTableCheck('wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8'::REGCLASS, 'Long table name could cause sequence collision while renaming #325');
+DROP TABLE "wadus_table_9473e8f6-2da1-11e8-8bca-0204e4dfe4d8";
+
 -- TODO: table with existing custom-triggered the_geom
 
 DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

test/CDB_CartodbfyTableTest_expect

@@ -150,5 +150,8 @@ SET
 BEGIN
 DO
 ROLLBACK
+CREATE TABLE
+Long table name could cause sequence collision while renaming #325 cartodbfied fine
+DROP TABLE
 DROP FUNCTION
 DROP FUNCTION

test/CDB_GhostTables.sql

@@ -0,0 +1,52 @@
+-- Create user and enable Ghost tables trigger
+\set QUIET on
+SET client_min_messages TO error;
+SELECT CDB_EnableGhostTablesTrigger();
+CREATE ROLE "fulano" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "fulano";
+GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano";
+SELECT cartodb.CDB_Conf_SetConf('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+DELETE FROM cdb_conf WHERE key = 'invalidation_service';
+SET SESSION AUTHORIZATION "fulano";
+SET client_min_messages TO notice;
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called (configuration not found)
+
+-- Add TIS configuration
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT cartodb.CDB_Conf_SetConf('invalidation_service', '{"host": "fake-tis-host", "port": 3142}');
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called
+
+BEGIN;
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0
+CREATE TABLE tmp(id INT);
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1
+END; -- _CDB_LinkGhostTables called
+
+-- Disable Ghost tables trigger
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableGhostTablesTrigger();
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists
+DROP TABLE tmp; -- _CDB_LinkGhostTables not called
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano";
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano";
+REVOKE ALL ON SCHEMA cartodb FROM "fulano";
+DROP ROLE "fulano";
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service';
+\set QUIET off

test/CDB_GhostTables_expect

@@ -0,0 +1,20 @@
+
+
+WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+BEGIN
+cdb_ddl_execution
+0
+CREATE TABLE
+1
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+COMMIT
+
+
+DROP TABLE

test/CDB_GreatCircle_expect

@@ -1,2 +1,2 @@
-LINESTRING(4.259 55.858,5.6692453115051 56.0150275120673,7.10720375678704 56.157400475677,8.5718366560563 56.2842986378254,10.0619272412891 56.3949153508462,11.5760785994189 56.4884642014437,13.1127142001617 56.564185934303,14.6700812655504 56.6213555706215,16.2462571744128 56.6592896061102,17.8391590143095 56.6773531596105,19.4465562981665 56.6749669334121,21.0660867567155 56.6516138405427,22.6952750058883 56.6068451534252,24.3315537765309 56.540286032869,25.9722872888145 56.4516403065472,27.6147962622065 56.3406943817481,29.2563839799455 56.207320197769,30.8943627796619 56.0514771479657,32.5260803224591 55.8732129290618,34.1489450028345 55.6726633044968,35.7604499005266 55.4500507979281,37.3581947399686 55.2056823610616,38.9399054089486 54.9399460854786,40.5034506895044 54.6533070499613,42.0468559644411 54.3463024122038,43.5683137754523 54.0195358662507,45.066191217402 53.673671594382,46.5390342525062 53.3094278446298,47.9855691138079 52.9275702630659,49.4047010366934 52.5289051040742,50.7955106088955 52.1142724327331,52.1572480633875 51.6845394219955,53.4893258557794 51.2405938343407,54.7913098701049 50.7833377637432,56.0629095865715 50.3136816997865,57.3039675245588 49.8325389621027,58.5144482465496 49.3408205404538,59.6944271762829 48.8394303639846,60.8440794494795 48.329261012675,61.9636689799149 47.811189874886,63.0535378889196 47.2860757471582,64.1140964137264 46.7547558660385,65.1458133802427 46.2180433565989,66.1492072992903 45.676725078361,67.1248381223566 45.131559846414,68.0732996734468 44.583277003498,68.9952127576034 44.0325753175597,69.8912189338183 43.4801221786776,70.7619749300985 42.9265530691612,71.6081476710291 42.3724712809595,72.4304098829428 41.8184478551838,73.2294362384225 41.2650217194684,74.0059 40.7127)
-LINESTRING(4.259 55.858,4.96060044865294 55.9382939511593,5.6692453115051 56.0150275120673,6.38482117645567 56.0880973218335,7.10720375678705 56.157400475677,7.83625773770865 56.2228347173136,8.5718366560563 56.2842986378254,9.31378281572326 56.3416918804739,10.0619272412891 56.3949153508462,10.8160896721679 56.4438714316548,11.5760785994189 56.4884642014437,12.3416913471456 56.528599656387,13.1127142001617 56.5641859343031,13.8889225793161 56.5951335399513,14.6700812655504 56.6213555706215,15.4559446734179 56.6427679409819,16.2462571744128 56.6592896061102,17.0407534700619 56.6708427815999,17.8391590143095 56.6773531596105,18.6411904842936 56.6787501197174,19.4465562981665 56.6749669334121,20.2549571781681 56.6659409611101,21.0660867567155 56.6516138405428,21.8796322228404 56.6319316654367,22.6952750058883 56.6068451534252,23.5126914929996 56.5763098021872,24.3315537765309 56.5402860328691,25.1515304272452 56.4987393199198,25.9722872888145 56.4516403065472,26.7934882889404 56.3989649050969,27.6147962622065 56.3406943817482,28.4358737796488 56.2768154250305,29.2563839799456 56.207320197769,30.0759913971174 56.1322063721813,30.8943627796619 56.0514771479657,31.7111678961496 55.9651412533344,32.5260803224592 55.8732129290618,33.3387782060384 55.7757118957345,34.1489450028345 55.6726633044969,34.9562701828379 55.5640976716962,35.7604499005266 55.4500507979282,36.5611876268714 55.3305636720814,37.3581947399687 55.2056823610617,38.1511910717861 55.0754578859583,38.9399054089486 54.9399460854787,39.7240759459355 54.7992074675415,40.5034506895044 54.6533070499613,41.277787813601 54.5023141912026,42.0468559644411 54.3463024122039,42.8104345158644 54.1853492102971,43.5683137754524 54.0195358662508,44.3202951422663 53.8489472454711,45.066191217402 53.6736715943821,45.8058258688602 53.4938003329924,46.5390342525062 53.3094278446299,47.2656627911282 53.1206512637978,47.985569113808 52.9275702630661,48.6986219579803 52.7302868398744,49.4047010366934 52.5289051040743,50.1036968736777 52.3235310669909,50.7955106088955 52.1142724327332,51.4800537772815 51.9012383924278,52.1572480633875 51.6845394219956,52.8270250346284 51.4642870840378,53.4893258557795 51.2405938343408,54.1441009873167 51.0135728334539,54.791309870105 50.7833377637434,55.4309205988438 50.5500026522693,56.0629095865715 50.3136816997866,56.6872612224038 50.0744891161178,57.3039675245588 49.8325389621028,57.9130277905821 49.5879449982851,58.5144482465496 49.3408205404539,59.1082416968843 49.091278322122,59.6944271762829 48.8394303639847,60.2730296051101 48.5853878503721,60.8440794494795 48.3292610126751,61.40761238711 48.0711590197009,61.9636689799149 47.8111898748862,62.5122943541616 47.5494603202768,63.0535378889195 47.2860757471584,63.5874529134047 47.0211401132109,64.1140964137264 46.7547558660387,64.6335287494427 46.4870238729191,65.1458133802426 46.2180433565991,65.6510166029904 45.9479118369597,66.1492072992903 45.6767250783612,66.6404566936622 45.4045770424768,67.1248381223566 45.1315598464143,67.6024268127789 44.8577637259253,68.0732996734467 44.5832770034983,68.5375350943572 44.3081860611283,68.9952127576034 44.0325753175599,69.4464134580461 43.7565272097974,69.8912189338183 43.4801221786779,70.3297117064144 43.2034386583077,70.7619749300985 42.9265530691615,71.1880922503468 42.6495398146493,71.6081476710291 42.3724712809597,72.0222254300221 42.0954178399905,72.4304098829428 41.8184478551841,72.8327853946822 41.5416276900883,73.2294362384225 41.2650217194687,73.6204465018146 40.9886923428039,74.0059 40.7127)
+LINESTRING(4.259 55.858,5.53349240387128 56.0006659105918,6.81698919498694 56.130094578525,8.10870381314147 56.2461317260662,9.40781156033233 56.3486370295466,10.7134527044527 56.4374849223869,12.0247359780093 56.5125653297878,13.3407424424749 56.573784325367,14.660529681225 56.6210647008095,15.9831362768927 56.654346440595,17.307586522649 56.67358709506,18.6328953115992 56.6787620464102,19.9580731443722 56.6698646638042,21.282131192215 56.6469063452276,22.6040863516019 56.6099164455407,23.922966226566 56.5589420917603,25.2378139766594 56.4940478882858,26.5476929715805 56.4153155163602,27.8516911979552 56.3228432335229,29.1489253693643 56.2167452801302,30.4385446972665 56.0971512011646,31.7197342877491 55.9642050924945,32.9917181368037 55.8180647814723,34.2537617048216 55.6589009522625,35.5051740589896 55.4868962265697,36.7453095800251 55.3022442104976,37.9735692370026 55.1051485181267,39.1894014407465 54.8958217820713,40.3923024922398 54.6744846607816,41.5818166476476 54.4413648517294,42.757535825811 54.1966961188706,43.919098987406 53.9407173419567,45.0661912174019 53.673671594382,46.198542544017 53.3958052553427,47.3159265281308 53.1073671611612,48.4181586571351 52.8086077997244,49.5050945765883 52.499778551104,50.5766281918714 52.1811309766006,51.6326896704254 51.8529161576737,52.673243373185 51.5153840855177,53.6982857415906 51.1687831014009,54.7078431641625 50.8133593873253,55.7019698441171 50.4493565060761,56.6807456869812 50.0770149893128,57.6442742246566 49.6965719720156,58.5926805899637 49.3082608713202,59.5261095533829 48.9123111075629,60.4447236315382 48.5089478652008,61.3487012749643 48.0983918911668,62.2382351408597 47.6808593281578,63.1135304548766 47.2565615803358,63.9748034645285 46.8257052089336,64.822279985501 46.3884918552974,65.6561940410346 45.9451181889661,66.476786593589 45.4957758784676,67.284304367196 45.0406515826125,68.0789987582454 44.5799269601738,68.8611248319107 44.1137786959568,69.6309404010034 43.6423785413868,70.388705183725 43.1658933678633,71.1346800365587 42.6844852312539,71.8691262583921 42.1983114460249,72.5923049618788 41.7075246676227,73.3044765080245 41.2122729818388,74.0059 40.7127)
+LINESTRING(4.259 55.858,4.89507305967085 55.930977446384,5.53349240387128 56.0006659105918,6.17416348361598 56.0670448594645,6.81698919498694 56.130094578525,7.46186995983655 56.1897961993418,8.10870381314147 56.2461317260662,8.75738649688733 56.2990840610623,9.40781156033233 56.3486370295466,10.0598704664666 56.3947754031591,10.7134527044527 56.4374849223869,11.3684459078018 56.4767523177655,12.0247359780093 56.5125653297878,12.6822072133468 56.5449127274491,13.3407424424749 56.573784325367,14.0002231625192 56.5991709994144,14.660529681225 56.6210647008095,15.3215412627822 56.6394584686143,15.9831362768927 56.654346440595,16.6451923506331 56.6657238624055,17.307586522649 56.67358709506,17.9701953992046 56.677933620668,18.6328953115992 56.6787620464102,19.2955624744544 56.6760721067401,19.9580731443722 56.6698646638042,20.6203037784591 56.6601417060788,21.282131192215 56.6469063452276,21.943432716288 56.6301628111935,22.6040863516019 56.6099164455407,23.2639709223762 56.5861736930735,23.922966226566 56.5589420917603,24.5809531832687 56.5282302610022,25.2378139766594 56.4940478882858,25.8934321960358 56.4564057142701,26.5476929715805 56.4153155163602,27.2004831054654 56.3707900908252,27.8516911979552 56.3228432335229,28.5012077681911 56.2714897192993,29.1489253693643 56.2167452801302,29.7947386980206 56.1586265820819,30.4385446972665 56.0971512011646,31.0802426536785 56.0323375981587,31.7197342877491 55.9642050924945,32.3569238377352 55.8927738352675,32.9917181368037 55.8180647814723,33.6240266834038 55.74009966154,34.2537617048216 55.6589009522625,34.8808382139074 55.5744918471876,35.5051740589896 55.4868962265697,36.1266899670207 55.3961386269571,36.7453095800251 55.3022442104976,37.3609594849451 55.2052387340427,37.9735692370026 55.1051485181267,38.5830713767178 55.0020004158976,39.1894014407465 54.8958217820713,39.7924979667135 54.7866404419798,40.3923024922398 54.6744846607816,40.9887595483734 54.5593831128969,41.5818166476476 54.4413648517294,42.1714242670021 54.3204592797319,42.757535825811 54.1966961188706,43.3401076592708 54.0701053815371,43.919098987406 53.9407173419567,44.4944718799548 53.8085625081347,45.0661912174019 53.673671594382,45.6342246484243 53.5360754944551,46.198542544017 53.3958052553427,46.7591179485663 53.2528920517295,47.3159265281308 53.1073671611612,47.8689465161932 52.9592619399335,48.4181586571351 52.8086077997244,48.9635461476859 52.6554361849853,49.5050945765883 52.499778551104,50.0427918627159 52.3416663433486,50.5766281918714 52.1811309766006,51.1065959524853 52.0182038158815,51.6326896704254 51.8529161576737,52.1549059431199 51.6852992120372,52.673243373185 51.5153840855177,53.1877025017413 51.3432017648431,53.6982857415906 51.1687831014009,54.2049973104167 50.9921587964881,54.7078431641625 50.8133593873253,55.2068309307272 50.6324152338211,55.7019698441171 50.4493565060761,56.1932706791714 50.2642131726125,56.6807456869812 50.0770149893128,57.1644085311015 49.8877914890534,57.6442742246566 49.6965719720156,58.1203590684218 49.5033854966561,58.5926805899637 49.3082608713202,59.0612574839055 49.1112266464775,59.5261095533829 48.9123111075629,59.9872576527434 48.7115422684016,60.4447236315382 48.5089478652008,60.8985302798459 48.3045553510865,61.3487012749643 48.0983918911668,61.7952611294973 47.8904843581013,62.2382351408597 47.6808593281578,62.6776493422177 47.4695430777354,63.1135304548766 47.2565615803358,63.5459058421237 47.0419405039633,63.9748034645285 46.8257052089336,64.4002518367009 46.6078807460742,64.822279985501 46.3884918552974,65.2409174096934 46.1675629645276,65.6561940410346 45.9451181889661,66.0681402067793 45.7211813306754,66.476786593589 45.4957758784676,66.8821642128236 45.2689250080781,67.284304367196 45.0406515826125,67.6832386187654 44.8109781532476,68.0789987582454 44.5799269601738,68.4716167756018 44.3475199337644,68.8611248319107 44.1137786959568,69.2475552324516 43.878724561833,69.6309404010034 43.6423785413868,70.0113128553159 43.4047613414637,70.388705183725 43.1658933678633,70.7631500228809 42.925794727592,71.1346800365587 42.6844852312539,71.5033278955199 42.4419843955718,71.8691262583921 42.1983114460249,72.2321077535378 41.953485319597,72.5923049618788 41.7075246676227,72.9497504006463 41.4604478587259,73.3044765080245 41.2122729818388,73.6565156286596 40.963017849297,74.0059 40.7127)

test/CDB_HelperTest.sql

@@ -126,3 +126,13 @@ SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 6);
 
 -- Test _CDB_Octet_Truncate UTF8 case
 SELECT * FROM cartodb._CDB_Octet_Truncate('piraña', 7);
+
+-- Test _CDB_Table_Exists
+CREATE TABLE public.this_table_exists();
+SELECT cartodb._CDB_Table_Exists('this_table_does_not_exist');
+SELECT cartodb._CDB_Table_Exists('this_schema_does_not_exist.this_table_does_not_exist');
+SELECT cartodb._CDB_Table_Exists('this_table_exists');
+SELECT cartodb._CDB_Table_Exists('public.this_table_exists');
+SELECT cartodb._CDB_Table_Exists('raster_overviews'); -- view created by postgis
+SELECT cartodb._CDB_Table_Exists('public.raster_overviews');
+DROP TABLE public.this_table_exists

test/CDB_HelperTest_expect

@@ -57,3 +57,11 @@ DROP TABLE
 pira
 pirañ
 piraña
+CREATE TABLE
+f
+f
+t
+t
+f
+f
+DROP TABLE

test/CDB_HexagonTest.sql

@@ -1,3 +1,6 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
 -- Check correctness of an hexagons grid
 --
 -- Cells must have no overlaps and have a number of
@@ -45,3 +48,9 @@ WITH
  0.002 as radius ),
  grid AS ( SELECT CDB_HexagonGrid(env, radius) AS cell from params)
 SELECT '#160', count(cell) > 23000 from grid;
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_HexagonGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1);

test/CDB_HexagonTest_expect

@@ -1,2 +1,5 @@
+SET
 9|63|count / npoints
 #160|t
+3886
+ERROR:  The requested grid is too big to be rendered

test/CDB_JenksBinsTest.sql

@@ -1,11 +1,36 @@
 WITH data AS (
-    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x
-        WHERE x % 5 != 0 AND x % 7 != 0
-    ) 
-SELECT unnest(CDB_JenksBins(s, 7)) FROM data;
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5)) FROM data;
+
 
 WITH data_nulls AS (
-    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,300) x
-        WHERE x % 5 != 0 AND x % 7 != 0
-    )
-SELECT unnest(CDB_JenksBins(s, 7)) FROM data_nulls;
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 null, null,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 null, null,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5)) FROM data_nulls;
+
+
+WITH data_inverse AS (
+    SELECT Array[0.99, 1.0, 1.01,
+                 4.99, 5.01,
+                 10.01, 10.01,
+                 15.01, 14.99,
+                 20.1, 19.9]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 5, 0, true)) FROM data_inverse;
+
+
+WITH data_small AS (
+    SELECT Array[0.99, 1.0, 10.01, 10.01, 10.01, 10.01]::numeric[] AS s
+)
+SELECT unnest(CDB_JenksBins(s, 4)) FROM data_small;

test/CDB_JenksBinsTest_expect

@@ -1,14 +1,18 @@
-43
-86
-129
-172
-213
-257
-299
-37
-51
-97
-157
-213
-241
-
+1.01
+5.01
+10.01
+15.01
+20.1
+1.01
+5.01
+10.01
+15.01
+20.1
+0.99
+4.99
+10.01
+14.99
+19.9
+0.99
+1.0
+10.01

test/CDB_OverviewsTest.sql

@@ -2,6 +2,7 @@ SET client_min_messages TO error;
 \set VERBOSITY default
 
 \i test/overviews/fixtures.sql
+vacuum ANALYZE; -- Make sure there are metrics for ST_EstimatedExtent
 
 SELECT _CDB_Aggregable_Attributes_Expression('base_bare_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);

test/CDB_OverviewsTest_expect

@@ -6,6 +6,7 @@ INSERT 0 1114
 CREATE TABLE
 INSERT 0 5
 SELECT 1114
+VACUUM
 
 
 

test/CDB_QuantileBinsTest.sql

@@ -1,11 +1,17 @@
 WITH data AS (
-    SELECT array_agg(x::numeric) s FROM generate_series(1,100) x
-        WHERE x % 5 != 0 AND x % 7 != 0
+    SELECT array_agg(x::numeric) AS s
+    FROM generate_series(0, 99) AS x
     ) 
-SELECT unnest(CDB_QuantileBins(s, 7)) FROM data;
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data;
 
 WITH data_nulls AS (
-    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
-        WHERE x % 5 != 0 AND x % 7 != 0
+    SELECT array_agg(x::numeric) AS s
+      FROM (
+        SELECT x FROM generate_series(0, 99) AS x
+        UNION ALL
+        SELECT null AS x FROM generate_series(1, 10) AS x
+        ) _wrap
     )
-SELECT unnest(CDB_QuantileBins(s, 7)) FROM data_nulls;
+SELECT unnest(CDB_QuantileBins(s, 10))
+  FROM data_nulls;

test/CDB_QuantileBinsTest_expect

@@ -1,14 +1,20 @@
-13
+9
+19
 29
-43
-57
-71
-86
+39
+49
+59
+69
+79
+89
 99
+9
+19
 29
-57
-87
-
-
-
+39
+49
+59
+69
+79
+89
 99

test/CDB_QueryTablesTest_expect

@@ -7,7 +7,7 @@ SELECT 1 as col1; select 2 as col2|{}
 WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
 ERROR:  relation "nonexistant" does not exist
 begin; select * from pg_class; commit;|{pg_catalog.pg_class}
-WARNING:  CDB_QueryTables cannot explain query: select * from test (42P01: relation "test" does not exist)
+WARNING:  CDB_QueryTables cannot explain query: insert into test values (1) (42P01: relation "test" does not exist)
 ERROR:  relation "test" does not exist
 WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
 CREATE SCHEMA

test/CDB_RectangleTest.sql

@@ -0,0 +1,8 @@
+set client_min_messages to error;
+\set VERBOSITY TERSE
+
+-- Check small grids are generated...
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 10, 10);
+
+-- But large grids produce an error
+SELECT COUNT(*) FROM cartodb.CDB_RectangleGrid(ST_MakeEnvelope(0,0,1000,1000,3857), 1, 1);

test/CDB_RectangleTest_expect

@@ -0,0 +1,3 @@
+SET
+10000
+ERROR:  The requested grid is too big to be rendered

test/CDB_Username.sql

@@ -0,0 +1,23 @@
+SELECT session_user; -- postgres
+SELECT CDB_Username(); -- (NULL)
+
+-- Add the role fulano with api_key and connect with it
+\set QUIET on
+CREATE ROLE fulano LOGIN;
+GRANT USAGE ON SCHEMA cartodb TO fulano;
+GRANT EXECUTE ON FUNCTION CDB_Username() TO fulano;
+INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+SET SESSION AUTHORIZATION fulano;
+\set QUIET off
+
+SELECT session_user; -- fulano
+SELECT CDB_Username(); -- fulanito
+
+-- Remove fulano
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE USAGE ON SCHEMA cartodb FROM fulano;
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano;
+DROP ROLE fulano;
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano';
+\set QUIET off

test/CDB_Username_expect

@@ -0,0 +1,4 @@
+postgres
+
+fulano
+fulanito

test/extension/test.sh

@@ -418,6 +418,36 @@ function test_cdb_tablemetadatatouch_fully_qualifies_names() {
     sql postgres 'DROP TABLE touch_invalidations'
 }
 
+function test_cdb_tablemetadata_text() {
+
+    #create and touch tables
+    sql "CREATE TABLE touch_ex_a (id int);"
+    sql "CREATE TABLE touch_ex_b (id int);"
+    sql "CREATE TABLE touch_ex_c (id int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_a');"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_b');"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_ex_c');"
+
+    #ensure there is 1 record per table
+    QUERY="SELECT COUNT(1) FROM (SELECT 1 FROM cdb_tablemetadata_text "
+    QUERY+="GROUP BY tabname HAVING COUNT(1) > 1) s;"
+    sql postgres "$QUERY" should "0"
+
+    #ensure timestamps are distinct and properly ordered
+    QUERY="SELECT (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_a')"
+    QUERY+="    < (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_b');"
+    sql postgres "$QUERY" should "t"
+    QUERY="SELECT (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_b')"
+    QUERY+="    < (SELECT updated_at FROM CDB_TableMetadata_Text WHERE tabname='public.touch_ex_c');"
+    sql postgres "$QUERY" should "t"
+
+    #cleanup
+    sql "DROP TABLE touch_ex_a;"
+    sql "DROP TABLE touch_ex_b;"
+    sql "DROP TABLE touch_ex_c;"
+
+}
+
 function test_cdb_column_names() {
     sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
     sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
@@ -543,7 +573,12 @@ END
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
 
-    sql postgres "SELECT cartodb.CDB_Conf_SetConf('fdws', '{\"test_fdw\": {\"server\": {\"host\": \"localhost\", \"dbname\": \"fdw_target\"},
+    # Add PGPORT to conf if it is set
+    PORT_SPEC=""
+    if [[ "$PGPORT" != "" ]] ; then
+        PORT_SPEC=", \"port\": \"$PGPORT\""
+    fi
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('fdws', '{\"test_fdw\": {\"server\": {\"host\": \"localhost\", \"dbname\": \"fdw_target\" $PORT_SPEC },
                                            \"users\": {\"public\": {\"user\": \"fdw_user\", \"password\": \"foobarino\"}}}}')"
 
     sql postgres "SELECT cartodb._CDB_Setup_FDW('test_fdw')"

test/organization/test.sh

@@ -324,6 +324,34 @@ function test_member_2_can_write_to_member_1_table_after_write_permission_is_add
     sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
 }
 
+function test_member_2_can_write_to_member_1_table_and_sequence_after_write_permission_is_added() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id SERIAL NOT NULL UNIQUE;"
+
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
+}
+
+function test_member_2_can_write_to_member_1_table_with_non_sequence_cartodb_id_after_write_permission_is_added() {
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo ADD cartodb_id INTEGER;"
+
+    sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Write_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_1.foo VALUES (5), (6), (7), (8), (9);'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 10
+    sql cdb_testmember_2 'DELETE FROM cdb_testmember_1.foo where a = 9;'
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 9
+
+    sql cdb_testmember_1 "ALTER TABLE cdb_testmember_1.foo DROP cartodb_id;"
+}
+
 function test_member_1_removes_access_and_member_2_can_no_longer_query_the_table() {
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Add_Table_Read_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2')"
     sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_1.foo;' should 5
@@ -398,6 +426,12 @@ function test_cdb_querytables_returns_schema_and_table_name() {
     sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
 }
 
+function test_cdb_querytables_works_with_parentheses() {
+    load_sql_file scripts-available/CDB_QueryStatements.sql
+    load_sql_file scripts-available/CDB_QueryTables.sql
+    sql cdb_testmember_1 "select * from CDB_QueryTables('(select * from foo)');" should "{cdb_testmember_1.foo}"
+}
+
 function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
     load_sql_file scripts-available/CDB_QueryStatements.sql
     load_sql_file scripts-available/CDB_QueryTables.sql
@@ -418,7 +452,7 @@ function test_cdb_usertables_should_work_with_orgusers() {
     sql cdb_testmember_1 "CREATE TABLE test_perms_pub (a int)"
     sql cdb_testmember_1 "INSERT INTO test_perms_pub (a) values (1);"
     sql cdb_testmember_1 "GRANT SELECT ON TABLE test_perms_pub TO publicuser"
-    
+
     sql cdb_testmember_1 "CREATE TABLE test_perms_priv (a int)"