Release 0.14.4

Update CartoDB User table docs and minor edits on cartodbfy reqs

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.13.0
+EXTVERSION = 0.14.4
 
 SED = sed
 
@@ -55,9 +55,15 @@ UPGRADABLE = \
   0.11.2 \
   0.11.3 \
   0.11.4 \
-	0.11.5 \
-	0.12.0 \
-	0.13.0 \
+  0.11.5 \
+  0.12.0 \
+  0.13.0 \
+  0.13.1 \
+  0.14.0 \
+  0.14.1 \
+  0.14.2 \
+  0.14.3 \
+	0.14.4 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -91,7 +97,8 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
     $(SED) -e 's/public\./cartodb./g' \
-        -e 's/:DATABASE_USERNAME/cdb_org_admin/g' >> $@
+        -e 's/:DATABASE_USERNAME/cdb_org_admin/g' \
+        -e "s/''public''/''cartodb''/g" >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 
@@ -110,6 +117,9 @@ $(EXTENSION).control: $(EXTENSION).control.in Makefile
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
+# Needed for consistent `echo` results with backslashes
+SHELL = bash
+
 legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
 	mkdir -p expected/test/
@@ -117,14 +127,14 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	for f in $(REGRESS_OLD); do \
     tn=`basename $${f} .sql`; \
     of=sql/test/$${tn}.sql; \
-    echo '\\set ECHO none' > $${of}; \
-    echo '\\a' >> $${of}; \
-    echo '\\t' >> $${of}; \
-    echo '\\set QUIET off' >> $${of}; \
+    echo '\set ECHO none' > $${of}; \
+    echo '\a' >> $${of}; \
+    echo '\t' >> $${of}; \
+    echo '\set QUIET off' >> $${of}; \
     cat $${f} | \
       $(SED) -e 's/public\./cartodb./g' >> $${of}; \
     exp=expected/test/$${tn}.out; \
-    echo '\\set ECHO none' > $${exp}; \
+    echo '\set ECHO none' > $${exp}; \
     cat test/$${tn}_expect >> $${exp}; \
   done
 

NEWS.md

@@ -1,4 +1,30 @@
+0.14.4 (2016-03-29)
+* Fix creating overviews for tables with boolean columns
+  [#214](https://github.com/CartoDB/cartodb-postgresql/pull/214)
+* Fix tests for some systems [#215](https://github.com/CartoDB/cartodb-postgresql/pull/215)
+
+0.14.3 (2016-03-17)
+-------------------
+* Fix for `cartodb_id` bigint casting hardcoded in 0.14.2 to support `cartodb_id` text columns [#210](https://github.com/CartoDB/cartodb-postgresql/pull/210)
+
+0.14.2 (2016-03-15)
+-------------------
+* Support text `cartodb_id` columns in `_CDB_Has_Usable_Primary_ID` [#202](https://github.com/CartoDB/cartodb-postgresql/pull/202)
+
+0.14.1 (2016-03-07)
+-------------------
+* Fully qualify table names in cache cdb_invalidate_varnish calls [#198](https://github.com/CartoDB/cartodb-postgresql/issues/198)
+
+0.14.0 (2016-02-14)
+-------------------
+* Add CDB_ForeignTable.sql to support FDW's [#199](https://github.com/CartoDB/cartodb-postgresql/pull/199)
+
+0.13.1 (2016-02-01)
+-------------------
+* Fix migration fron unpackaged. [193](https://github.com/CartoDB/cartodb-postgresql/pull/193)
+
 0.13.0 (2016-01-29)
+-------------------
 * Add CDB_CreateOverviews, CDB_DropOverviews and CDB_Overviews for vector overviews support. [185](https://github.com/CartoDB/cartodb-postgresql/pull/185)
 * Convert some simple functions from plpgsql to sql. [188](https://github.com/CartoDB/cartodb-postgresql/pull/188)
 

README.md

@@ -1,7 +1,7 @@
 cartodb-postgresql
 ==================
 
-[![Build Status](http://travis-ci.org/CartoDB/cartodb-postgresql.png)]
+[![Build Status](http://api.travis-ci.org/CartoDB/cartodb-postgresql.svg?branch=master)]
 (http://travis-ci.org/CartoDB/cartodb-postgresql)
 
 PostgreSQL extension for CartoDB

doc/CartoDB-user-table.md

@@ -1,29 +0,0 @@
-A "cartodb" user table is a table with a well-known set of fields and a well-known set of triggers attached on. 
-
-The fields are:
-
- - `cartodb_id`, a numerical primary key of serial type
- - `created_at`, timestamp with timezone not null default now()
- - `updated_at`, timestamp with timezone not null default now()
- - `the_geom`, geometry, GiST indexed, constrained (see below)
- - `the_geom_webmercator`, geometry, GiST indexed, constrained (see below)
-
-The values of "the_geom" and "the_geom_webmercator" must match these constraints:
-
- - Only POINT, MULTILINE, MULTIPOLYGON types ? Maybe UNCONSTRAINED
- - Only 2 dimensions ? Maybe UNCONSTRAINED
- - SRID=4326 for the_geom and SRID=3857 for the_geom_webmercator
-
-The triggers are:
-
- - `track_updates` after modifying statement updates cdb_tablemetadata
- - `test_quota` before changing statement to forbid if overquota
- - `test_quota_per_row` before changing row to forbod if overquota (checked on a probabilistic basis)
- - `update_the_geom_webmercator` before insert or update row to maintain the_geom_webmercator
- - `update_updated_at_trigger` before update row to maintain updated_at
-
-Some conversions will be attempted to perform upon cartodbfication when certain fields appear:
-
- - `cartodb_id`: If found type TEXT will be attempted to cast
- - `created_at`: If found type TEXT will be attempted to cast
- - `updated_at`: If found type TEXT will be attempted to cast

doc/CartoDB-user-table.rst

@@ -0,0 +1,68 @@
+CartoDB User Table
+==================
+
+Introduction
+----------
+A CartoDB user table is a table with a well-known set of columns and a well-known set of triggers attached on.
+
+Columns
+----------
+The required columns of a CartoDB table are:
+
+-  ``cartodb_id``
+  - This column will be used as the primary key of the table and it has a sequence as default value
+  - Its values must be integer, non-zero, non-null and unique
+  -  B-Tree indexed
+-  ``the_geom``
+  - This column stores the main geometric features of a table
+  - The type of the column in the Postgres database is ``geometry(Geometry,4326)```
+  - GiST indexed
+  -  geometry, GiST indexed, constrained (see below)
+-  ``the_geom_webmercator``
+  - This column stores the geometries used for rendering purposes
+  - The type of the column in the Postgres database is ``geometry(Geometry,3857)``
+  - GiST indexed
+  - This column is automatically updated by the system when the ``the_geom`` column is updated or when there is an insertion of a new row into the table (See triggers below)
+
+The values of ``the_geom`` and ``the_geom_webmercator`` must be two-dimensional Points, MultiLineStrings or MultiPolygons. Different geometric types in a CartoDB table are not supported.
+
+Described table example
+^^^^^^^^^^
+::
+
+        Column        |          Type           |                       Modifiers                        
+  ----------------------+-------------------------+--------------------------------------------------------
+   cartodb_id           | bigint                  | not null default nextval('t_cartodb_id_seq'::regclass)
+   the_geom             | geometry(Geometry,4326) | 
+   the_geom_webmercator | geometry(Geometry,3857) | 
+  Indexes:
+      "table_name_pkey" PRIMARY KEY, btree (cartodb_id)
+      "table_name_the_geom_idx" gist (the_geom)
+      "table_name_the_geom_webmercator_idx" gist (the_geom_webmercator)
+
+Triggers
+----------
+The triggers generated in each CartoDB table are:
+
+-  ``track_updates`` after modifying statement updates ``cdb_tablemetadata``
+-  ``test_quota`` before changing statement to forbid if overquota
+-  ``test_quota_per_row`` before insert ot update row to forbid if overquota (checked on a probabilistic basis)
+-  ``update_the_geom_webmercator`` before insert or update row to maintain the ``the_geom_webmercator`` updated with the contents in ``the_geom``
+
+Described triggers example
+^^^^^^^^^^
+::
+
+  test_quota BEFORE INSERT OR UPDATE ON t FOR EACH STATEMENT EXECUTE PROCEDURE cdb_checkquota('0.1', '-1', 'public')
+  test_quota_per_row BEFORE INSERT OR UPDATE ON t FOR EACH ROW EXECUTE PROCEDURE cdb_checkquota('0.001', '-1', 'public')
+  track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON t FOR EACH STATEMENT EXECUTE PROCEDURE cdb_tablemetadata_trigger()
+  update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON t FOR EACH ROW EXECUTE PROCEDURE _cdb_update_the_geom_webmercator()
+
+
+Further details
+----------
+
+Some conversions will be attempted to perform upon cartodbfication when certain fields appear:
+
+-  ``cartodb_id``: If found type TEXT will be attempted to cast to integer. If not casteable, an eror will be raised.
+-  ``the_geom``: If found type TEXT will be attempted to cast to geometry(Geometry,4326).

doc/cartodbfy-requirements.rst

@@ -1,59 +1,63 @@
+CartoDBfy Requirements
+======================
+
 Introduction
-============
+------------
 
-This document aims at describing what cartodbfy is and what its formal requirements are, with the following goals in mind:
+This document aims at describing what the CartoDBfication is and what its formal requirements are, with the following goals in mind:
 
-- clarify what are the expectations of the "cartodbfycation process".
-- define an important part of what should be a stable, public API
-- allow for better testing, which should in turn...
+- Clarify what are the expectations of the "cartodbfycation process".
+- Define an important part of what should be a stable, public API
+- Allow for better testing, which should in turn...
 - ...ease modifications and increase quality of the code
 
 
+What is the CartoDBfycation
+---------------------------
 
-What is the cartodbfycation
-===========================
-
-The cartodbfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
-
+The CartoDBfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
 
+It is performed by running the function ``CDB_CartodbfyTable(reloid REGCLASS)`` over a target table.
 
 Valid CartoDB tables
-====================
+--------------------
 
 A valid CartoDB table shall meet the following conditions:
 
-- Have a ``cartodb_id`` integer column as primary key with a sequence as default value
+- Have a ``cartodb_id`` column with integer, unique, non-zero and non-null values as primary key with a sequence as default value
 - Have a ``the_geom`` column of type ``Geometry`` with SRID 4326
 - Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857
-- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync
+- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync (task of the ``update_the_geom_webmercator`` trigger)
 
 Additionally, a CartoDB table can contain other columns.
 
+See the `CartoDB User Table documentation`_
 
+.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.md 
+for further information.
 
 High level requirements
-=======================
+-----------------------
 
 Here is a list of high level requirments for the public function ``CDB_CartodbfyTable()``:
 
-- A call to ``CDB_CartodbfyTable()`` shall modify/rewrite the table and produce a valid CartoDB table with the same name.
-- A call to ``CDB_CartodbfyTable()`` shall cause the registration of the table into the platform
-- It shall be idempotent, meaning that successive calls to ``CDB_CartodbfyTable()`` shall not produce any visible effect in the system.
+- A call to the function shall modify/rewrite the table and produce a valid CartoDB table with the same name.
+- A call to the function shall cause the registration of the table into the platform.
+- It shall be idempotent, meaning that successive calls to the function shall not produce any visible effect in the system.
 - If there's a column containing a geometry, it shall be used to generate ``the_geom`` and the ``the_geom_webmercator`` columns.
 - Exporting and re-importing the same table in CartoDB shall produce equivalent tables, with the same features associated to the same ``cartodb_id``'s.
 
-Note that there should be only one feature per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
 
+Note that there should be only one geometry per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
 
 
 Low-level requirements
-======================
+----------------------
 
-- If the original table contains a valid (unique and not null) ``cartodb_id`` column, it shall be used
-- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
+- If the original table contains a valid (integer, unique, non-zero and not null) ``cartodb_id`` column, it shall be used
+- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` geometric column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
 - A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata``
 - A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id``
 - A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator``
-- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns
-
+- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns (``cartodb_id``, ``the_geom``, ``the_geom_webmercator``)
 

scripts-available/CDB_CartodbfyTable.sql

@@ -470,6 +470,7 @@ $$ LANGUAGE 'plpgsql';
 -- If the table has both a usable key and usable geometry
 -- we can no-op on the table copy and just ensure that the 
 -- indexes and triggers are in place
+DROP FUNCTION IF EXISTS _CDB_Has_Usable_Primary_ID(reloid REGCLASS);
 CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Primary_ID(reloid REGCLASS)
   RETURNS BOOLEAN
 AS $$
@@ -489,114 +490,98 @@ BEGIN
   -- Do we already have a properly named column?
   SELECT a.attname, i.indisprimary, i.indisunique, a.attnotnull, a.atttypid
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_type t ON a.atttypid = t.oid
   LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
-  WHERE c.oid = reloid 
+  WHERE c.oid = reloid
   AND NOT a.attisdropped
   AND a.attname = const.pkey;
 
   -- Found something named right...
   IF FOUND THEN
-  
-    -- And it's an integer column...
-    IF rec.atttypid IN (20,21,23) THEN
-          
-      -- And it's a unique primary key! Done!
-      IF (rec.indisprimary OR rec.indisunique) AND rec.attnotnull THEN
-        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey);
-        RETURN true;
 
-      -- Check and see if the column values are unique and not null, 
-      -- if they are, we can use this column...
+    -- And it's a unique primary key! Done!
+    IF (rec.indisprimary OR rec.indisunique) AND rec.attnotnull THEN
+      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey);
+      RETURN true;
+
+    -- Check and see if the column values are unique and not null,
+    -- if they are, we can use this column...
+    ELSE
+
+      -- Assume things are OK until proven otherwise...
+      useable_key := true;
+      BEGIN
+        sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey);
+        sql := sql || ', ' || Format('ADD CONSTRAINT %s_integer CHECK (%s::integer >=0);', const.pkey, const.pkey);
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql;
+        EXECUTE sql;
+        EXCEPTION
+        -- Failed unique check...
+        WHEN unique_violation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
+          useable_key := false;
+        -- Failed not null check...
+        WHEN not_null_violation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey);
+          useable_key := false;
+        -- Failed integer check...
+        WHEN invalid_text_representation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('invalid input syntax for integer %s', const.pkey);
+          useable_key := false;
+        -- Other fatal error
+        WHEN others THEN
+          PERFORM _CDB_Error(sql, Format('_CDB_Has_Usable_Primary_ID: %s', SQLERRM));
+      END;
+
+      -- Clean up test constraint
+      IF useable_key THEN
+        PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey));
+        PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_integer', reloid::text, const.pkey));
+
+      -- Move non-valid column out of the way
       ELSE
 
-        -- Assume things are OK until proven otherwise...
-        useable_key := true;
-      
-        BEGIN
-          sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey);
-          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql;
-          EXECUTE sql;
-          EXCEPTION      
-          -- Failed unique check...
-          WHEN unique_violation THEN
-            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
-            useable_key := false;
-          -- Failed not null check...
-          WHEN not_null_violation THEN
-            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey);
-            useable_key := false;
-          -- Other fatal error
-          WHEN others THEN
-            PERFORM _CDB_Error(sql, '_CDB_Has_Usable_Primary_ID');          
-        END;
-  
-        -- Clean up test constraint
-        IF useable_key THEN
-          PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey));
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
+          Format('found non-valid ''%s''', const.pkey);
 
-        -- Move non-unique column out of the way
-        ELSE
-        
-          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
-            Format('found non-unique ''%s'', renaming it', const.pkey);
-
-          PERFORM _CDB_SQL(
-            Format('ALTER TABLE %s RENAME COLUMN %s TO %I',
-              reloid::text, rec.attname,
-              cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)),
-            '_CDB_Has_Usable_Primary_ID');
-        
-        END IF;
-        
-        return useable_key;
+        PERFORM _CDB_Error(sql, Format('_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, %s', const.pkey));
 
       END IF;
-    
-    -- It's not an integer column, we have to rename it
-    ELSE
-  
-      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 
-        Format('found non-integer ''%s'', renaming it', const.pkey);
 
-      PERFORM _CDB_SQL(
-        Format('ALTER TABLE %s RENAME COLUMN %s TO %I',
-                reloid::text, rec.attname, cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)),
-                '_CDB_Has_Usable_Primary_ID');
-    
+      RETURN useable_key;
+
     END IF;
-    
+
   -- There's no column there named pkey
   ELSE
 
-    -- Is there another suitable primary key already?
+    -- Is there another integer suitable primary key already?
     SELECT a.attname
     INTO rec
     FROM pg_class c 
-    JOIN pg_attribute a ON a.attrelid = c.oid 
+    JOIN pg_attribute a ON a.attrelid = c.oid
     JOIN pg_type t ON a.atttypid = t.oid
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     WHERE c.oid = reloid AND NOT a.attisdropped
     AND i.indisprimary AND i.indisunique AND a.attnotnull AND a.atttypid IN (20,21,23);
-  
+
     -- Yes! Ok, rename it.
     IF FOUND THEN
       PERFORM _CDB_SQL(Format('ALTER TABLE %s RENAME COLUMN %s TO %s', reloid::text, rec.attname, const.pkey),'_CDB_Has_Usable_Primary_ID');
       RETURN true;
     ELSE
-      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 
+      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
         Format('found no useful column for ''%s''', const.pkey);
     END IF;
-  
+
   END IF;
-  
+
   RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 'function complete';
 
   -- Didn't find re-usable key, so return FALSE
   RETURN false;
-
 END;
 $$ LANGUAGE 'plpgsql';
 
@@ -825,10 +810,10 @@ DECLARE
   str TEXT;
   table_srid INTEGER;
   geom_srid INTEGER;
-  
+
   has_usable_primary_key BOOLEAN;
   has_usable_pk_sequence BOOLEAN;
-  
+
 BEGIN
 
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', 'entered function';
@@ -839,7 +824,7 @@ BEGIN
   -- Save the raw schema/table names for later
   SELECT n.nspname, c.relname, c.relname
   INTO STRICT relschema, relname, destname
-  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid 
+  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
   WHERE c.oid = reloid;
 
   -- Default the destination to current schema if unspecified
@@ -890,7 +875,7 @@ BEGIN
 
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_geoms %', gc.has_usable_geoms;
 
-  -- We can only avoid a rewrite if both the key and 
+  -- We can only avoid a rewrite if both the key and
   -- geometry are usable
 
   -- No table re-write is required, BUT a rename is required to
@@ -934,7 +919,7 @@ BEGIN
 
   -- Add cartodb ID!
   IF has_usable_primary_key THEN
-    sql := sql || const.pkey;
+    sql := sql || const.pkey || '::integer ';
   ELSE
     sql := sql || 'nextval(''' || destseq || ''') AS ' || const.pkey;
   END IF;
@@ -1096,7 +1081,7 @@ BEGIN
 
   -- Run it!
   PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
-  
+
   -- Set up the primary key sequence
   -- If we copied the primary key from the original data, we need
   -- to set the sequence to the maximum value of that key

scripts-available/CDB_ForeignTable.sql

@@ -0,0 +1,199 @@
+---------------------------
+-- FDW MANAGEMENT FUNCTIONS
+--
+-- All the FDW settings are read from the `cdb_conf.fdws` entry json file.
+---------------------------
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text, config json)
+RETURNS void
+AS $$
+DECLARE
+  row record;
+  option record;
+  org_role text;
+BEGIN
+  -- This function tries to be as idempotent as possible, by not creating anything more than once
+  -- (not even using IF NOT EXIST to avoid throwing warnings)
+  IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
+    CREATE EXTENSION postgres_fdw;
+  END IF;
+  -- Create FDW first if it does not exist
+  IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
+    THEN
+    EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
+  END IF;
+
+  -- Set FDW settings
+  FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
+    LOOP
+      IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
+        THEN
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
+      ELSE
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
+      END IF;
+    END LOOP;
+
+    -- Create user mappings
+    FOR row IN SELECT p.key, p.value from lateral json_each(config->'users') p LOOP
+        -- Check if entry on pg_user_mappings exists
+
+        IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN
+          EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name);
+        END IF;
+
+    -- Update user mapping settings
+    FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP
+        IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN
+          EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value);
+        ELSE
+          EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value);
+        END IF;
+      END LOOP;
+    END LOOP;
+
+    -- Create schema if it does not exist.
+    IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
+      EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
+    END IF;
+
+    -- Give the organization role usage permisions over the schema
+    SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() INTO org_role;
+    EXECUTE FORMAT ('GRANT USAGE ON SCHEMA %I TO %I', fdw_name, org_role);
+
+    -- Bring here the remote cdb_tablemetadata
+    IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN
+      EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name);
+    END IF;
+    EXECUTE FORMAT ('GRANT SELECT ON %I.cdb_tablemetadata TO %I', fdw_name, org_role);
+
+END
+$$
+LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDWS()
+RETURNS VOID AS 
+$$
+DECLARE
+row record;
+BEGIN
+  FOR row IN SELECT p.key, p.value from lateral json_each(cartodb.CDB_Conf_GetConf('fdws')) p LOOP
+      EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING row.key, row.value;
+    END LOOP;
+  END
+$$
+LANGUAGE PLPGSQL;
+
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text)
+  RETURNS void AS
+$BODY$
+DECLARE
+config json;
+BEGIN
+  SELECT p.value FROM LATERAL json_each(cartodb.CDB_Conf_GetConf('fdws')) p WHERE p.key = fdw_name INTO config;
+  EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING fdw_name, config;
+END
+$BODY$
+LANGUAGE plpgsql VOLATILE;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_Add_Remote_Table(source text, table_name text)
+  RETURNS void AS
+$$
+BEGIN
+  PERFORM cartodb._CDB_Setup_FDW(source);
+  EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', source, table_name, source, source);
+  --- Grant SELECT to publicuser
+  EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO publicuser;', source, table_name);
+END
+$$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_Get_Foreign_Updated_At(foreign_table regclass)
+  RETURNS timestamp with time zone AS
+$$
+DECLARE
+  remote_table_name text;
+  fdw_schema_name text;
+  time timestamp with time zone;
+BEGIN
+  -- This will turn a local foreign table (referenced as regclass) to its fully qualified text remote table reference.
+  WITH a AS (SELECT ftoptions FROM pg_foreign_table WHERE ftrelid=foreign_table LIMIT 1),
+    b as (SELECT (pg_options_to_table(ftoptions)).* FROM a)
+    SELECT FORMAT('%I.%I', (SELECT option_value FROM b WHERE option_name='schema_name'), (SELECT option_value FROM b WHERE option_name='table_name'))
+  INTO remote_table_name;
+
+  -- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table.
+  SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name;
+  EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
+  RETURN time;
+END
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid)
+RETURNS TEXT AS $$
+    SELECT option_value FROM pg_options_to_table((
+
+        SELECT fs.srvoptions
+        FROM pg_foreign_table ft
+        LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid
+        WHERE ft.ftrelid = reloid
+
+    )) WHERE option_name='dbname';
+$$ LANGUAGE SQL;
+
+
+-- Return a set of (dbname, schema_name, table_name, updated_at)
+-- It is aware of foreign tables
+-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
+-- Note: dbname is never quoted whereas schema and table names are when needed.
+CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text)
+RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz)
+AS $$
+    WITH query_tables AS (
+      SELECT unnest(CDB_QueryTablesText(query)) schema_table_name
+    ), query_tables_oid AS (
+      SELECT schema_table_name, schema_table_name::regclass::oid AS reloid
+      FROM query_tables
+    ),
+    fqtn AS (
+      SELECT
+        (CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid)
+              ELSE current_database()
+         END)::text AS dbname,
+         quote_ident(n.nspname::text) schema_name,
+         quote_ident(c.relname::text) table_name,
+         c.relkind,
+         query_tables_oid.reloid
+      FROM query_tables_oid, pg_catalog.pg_class c
+      LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.oid = query_tables_oid.reloid
+    )
+    SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name,
+      (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
+            ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
+      END) AS updated_at
+    FROM fqtn;
+$$ LANGUAGE SQL;
+
+
+-- Return the last updated time of a set of tables
+-- It is aware of foreign tables
+-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
+CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[])
+RETURNS timestamptz AS $$
+    WITH t AS (
+        SELECT unnest(tables) AS schema_table_name
+    ), t_oid AS (
+        SELECT (t.schema_table_name)::regclass::oid as reloid FROM t
+    ), t_updated_at AS (
+        SELECT
+            (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
+                  ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
+             END) AS updated_at
+        FROM t_oid
+        LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
+    ) SELECT max(updated_at) FROM t_updated_at;
+$$ LANGUAGE SQL;

scripts-available/CDB_Overviews.sql

@@ -486,6 +486,8 @@ BEGIN
     -- Using 'string_agg(' || qualified_column || ',''/'')'
     -- here causes
     RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+  WHEN 'boolean' THEN
+    RETURN 'CASE count(*) WHEN 1 THEN BOOL_AND(' || qualified_column || ') ELSE NULL END::' || column_type;
   ELSE
     RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
   END CASE;
@@ -642,11 +644,7 @@ $$ LANGUAGE PLPGSQL;
 --                    created by the strategy must have the same columns
 --                    as the base table and in the same order.
 -- Return value: Array with the names of the generated overview tables
-CREATE OR REPLACE FUNCTION CDB_CreateOverviews(
-  reloid REGCLASS,
-  refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc,
-  reduce_strategy   regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc
-)
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
 RETURNS text[]
 AS $$
 DECLARE

scripts-available/CDB_TableMetadata.sql

@@ -5,6 +5,11 @@ CREATE TABLE IF NOT EXISTS
     updated_at timestamp with time zone not null default now()
   );
 
+CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS
+       SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at
+       FROM public.CDB_TableMetadata, pg_catalog.pg_class c
+       LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid;
+
 -- No one can see this
 -- Updates are only possible trough the security definer trigger
 -- GRANT SELECT ON public.CDB_TableMetadata TO public;
@@ -61,9 +66,11 @@ CREATE OR REPLACE FUNCTION _CDB_TableMetadata_Updated()
 RETURNS trigger AS
 $$
 DECLARE
-  tabname TEXT;
+  tabname regclass;
   rec RECORD;
   found BOOL;
+  schema_name TEXT;
+  table_name TEXT;
 BEGIN
 
   IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
@@ -98,9 +105,10 @@ BEGIN
                AND u.usesuper
              ORDER BY n.nspname
   LOOP
+    SELECT n.nspname, c.relname FROM pg_class c, pg_namespace n WHERE c.oid=tabname AND c.relnamespace = n.oid INTO schema_name, table_name;
     EXECUTE 'SELECT ' || quote_ident(rec.nspname) || '.'
             || quote_ident(rec.proname)
-            || '(' || quote_literal(tabname) || ')';
+            || '(' || quote_literal(quote_ident(schema_name) || '.' || quote_ident(table_name)) || ')';
     found := true;
     EXIT;
   END LOOP;
@@ -111,11 +119,11 @@ END;
 $$
 LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
 
-DROP TRIGGER IF EXISTS table_modified ON CDB_TableMetadata;
+DROP TRIGGER IF EXISTS table_modified ON public.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table
 --       oid (regclass) to its name
 CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
-ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
+ON public.CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
  _CDB_TableMetadata_Updated();
 
 

scripts-enabled/250-CDB_ForeignTable.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_ForeignTable.sql

test/CDB_CartodbfyTableTest.sql

@@ -1,5 +1,5 @@
 SET client_min_messages TO error;
-\set VERBOSITY default
+\set VERBOSITY terse
 
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
 RETURNS text AS
@@ -124,13 +124,21 @@ END;
 $$
 LANGUAGE 'plpgsql';
 
--- table with single non-geometrical column
+-- check cartodbfytable idempotence
 CREATE TABLE t AS SELECT 1::int as a;
 SELECT CDB_CartodbfyTable('public', 't'); -- should fail
 SELECT CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
 SELECT CDB_CartodbfyTableCheck('t', 'single non-geometrical column');
 DROP TABLE t;
 
+-- table with single non-geometrical column
+CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+DROP TABLE t;
+
 -- table with existing srid-unconstrained (but type-constrained) the_geom
 CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(0,0),4326)::geometry(point) as the_geom;
 SELECT CDB_CartodbfyTableCheck('t', 'srid-unconstrained the_geom');
@@ -164,19 +172,22 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
--- -- table with existing cartodb_id field of type text
--- CREATE TABLE t AS SELECT 10::text as cartodb_id;
--- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
--- select cartodb_id/2 FROM t;
--- DROP TABLE t;
+-- table with existing cartodb_id field of type text
+CREATE TABLE t AS SELECT 10::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+select cartodb_id/2 FROM t;
+DROP TABLE t;
 
--- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
--- -- table with existing cartodb_id field of type text not casting
--- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
--- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
--- select cartodb_id,_cartodb_id0 FROM t;
--- DROP TABLE t;
+-- table with existing cartodb_id field of type text not casting
+CREATE TABLE t AS SELECT 'nan'::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+DROP TABLE t;
+
+-- table with empty cartodb_id field of type text
+CREATE TABLE t AS SELECT null::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'empty text cartodb_id');
+SELECT cartodb_id from t;
+DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
@@ -291,7 +302,7 @@ INSERT INTO test VALUES
   (NULL),
   (3);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 -- Table with non unique cartodb_id
@@ -303,7 +314,7 @@ INSERT INTO test VALUES
   (2),
   (2);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 -- Table with non unique and null cartodb_id
@@ -316,7 +327,7 @@ INSERT INTO test VALUES
   (NULL),
   (2);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 CREATE TABLE test (

test/CDB_CartodbfyTableTest_expect

@@ -2,12 +2,16 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
 SELECT 1
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+DROP TABLE
+SELECT 1
 srid-unconstrained the_geom cartodbfied fine
 DROP TABLE
 SELECT 2
@@ -26,6 +30,17 @@ SELECT 1
 CREATE TRIGGER
 trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
+DROP TABLE
+SELECT 1
+text cartodb_id cartodbfied fine
+5
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+
 DROP TABLE
 SELECT 1
 unsequenced cartodb_id cartodbfied fine
@@ -77,30 +92,30 @@ Table with both the_geom and wkb_geometry #141 cartodbfied fine
 DROP TABLE
 CREATE TABLE
 INSERT 0 1
-Many colliding columns #141 cartodbfied fine
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: multiple primary keys for table "many_colliding_columns" are not allowed): ALTER TABLE many_colliding_columns ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
 DROP TABLE
 CREATE TABLE
 INSERT 0 4
-Table with null cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|
-4|3
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+3
 DROP TABLE
 CREATE TABLE
 INSERT 0 3
-Table with non unique cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|2
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+2
 DROP TABLE
 CREATE TABLE
 INSERT 0 4
-Table with non unique and null cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|
-4|2
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+2
 DROP TABLE
 CREATE TABLE
 CREATE INDEX

test/CDB_OverviewsTest.sql

@@ -20,14 +20,19 @@ SELECT count(*) FROM _vovw_5_base_t;
 
 SELECT CDB_CreateOverviews('polyg_t'::regclass);
 
+SELECT CDB_CreateOverviews('column_types_t'::regclass);
+
 SELECT CDB_Overviews('base_t'::regclass);
 SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
 SELECT CDB_Overviews('polyg_t'::regclass);
+SELECT CDB_Overviews('column_types_t'::regclass);
 
+SELECT CDB_DropOverviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('base_bare_t'::regclass);
 SELECT CDB_DropOverviews('base_t'::regclass);
 SELECT count(*) FROM _vovw_5_base_t;
 
+DROP TABLE column_types_t;
 DROP TABLE base_bare_t;
 DROP TABLE base_t;
 DROP TABLE polyg_t;

test/CDB_OverviewsTest_expect

@@ -5,6 +5,7 @@ CREATE TABLE
 INSERT 0 1114
 CREATE TABLE
 INSERT 0 5
+SELECT 1114
 
 
 
@@ -16,6 +17,7 @@ AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_
 {_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
 125
 
+{_vovw_5_column_types_t,_vovw_4_column_types_t,_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
@@ -34,6 +36,13 @@ AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_
 (base_t,3,_vovw_3_base_t)
 (base_t,4,_vovw_4_base_t)
 (base_t,5,_vovw_5_base_t)
+(column_types_t,0,_vovw_0_column_types_t)
+(column_types_t,1,_vovw_1_column_types_t)
+(column_types_t,2,_vovw_2_column_types_t)
+(column_types_t,3,_vovw_3_column_types_t)
+(column_types_t,4,_vovw_4_column_types_t)
+(column_types_t,5,_vovw_5_column_types_t)
+
 
 
 ERROR:  relation "_vovw_5_base_t" does not exist
@@ -42,3 +51,4 @@ LINE 1: SELECT count(*) FROM _vovw_5_base_t;
 DROP TABLE
 DROP TABLE
 DROP TABLE
+DROP TABLE

test/extension/test.sh

@@ -172,8 +172,7 @@ function drop_raster_table() {
     sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
 }
 
-
-function setup() {
+function setup_database() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
@@ -184,7 +183,10 @@ function setup() {
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
     # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
     ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
+}
 
+function setup() {
+    setup_database
 
     log_info "############################# SETUP #############################"
     create_role_and_schema cdb_testmember_1
@@ -199,6 +201,10 @@ function setup() {
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
 }
 
+
+function tear_down_database() {
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -219,9 +225,10 @@ function tear_down() {
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
 
-    ${CMD} -c "DROP DATABASE ${DATABASE}"
+    tear_down_database
 }
 
+
 function run_tests() {
     local FAILED_TESTS=()
 
@@ -343,6 +350,41 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_tablemetadatatouch_fully_qualifies_names() {
+    sql postgres "CREATE TABLE touch_invalidations (table_name text);"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+
+    #default schema
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "public.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+    sql postgres "DROP TABLE touch_example"
+
+    #setup different schema
+    sql postgres "CREATE SCHEMA test_schema;"
+    sql postgres "CREATE TABLE test_schema.touch_example (a int);"
+
+    #different schema outside search_path
+    sql postgres "SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #different schema in default search_path
+    sql postgres "SET search_path=test_schema,public,cartodb; SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #teardown different schema
+    sql postgres 'DROP TABLE test_schema.touch_example;'
+    sql postgres 'DROP SCHEMA test_schema;'
+
+
+
+    sql postgres 'DROP FUNCTION cartodb.cdb_invalidate_varnish(table_name text);'
+    sql postgres 'DROP TABLE touch_invalidations'
+}
+
 function test_cdb_column_names() {
     sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
     sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
@@ -429,6 +471,98 @@ function test_cdb_querytables_happy_cases() {
     sql postgres 'DROP SCHEMA foo;'
 }
 
+function test_foreign_tables() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_TableMetadata.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_ForeignTable.sql
+
+
+    DATABASE=fdw_target setup_database
+    ${CMD} -d fdw_target -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d fdw_target -f scripts-available/CDB_QueryTables.sql
+    ${CMD} -d fdw_target -f scripts-available/CDB_TableMetadata.sql
+
+    DATABASE=fdw_target sql postgres 'CREATE SCHEMA test_fdw;'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo (a) values (42);'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo2 (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo2 (a) values (42);'
+    DATABASE=fdw_target sql postgres "CREATE USER fdw_user WITH PASSWORD 'foobarino';"
+    DATABASE=fdw_target sql postgres 'GRANT USAGE ON SCHEMA test_fdw TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo2 TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON cdb_tablemetadata_text TO fdw_user;'
+
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('fdws', '{\"test_fdw\": {\"server\": {\"host\": \"localhost\", \"dbname\": \"fdw_target\"},
+                                           \"users\": {\"public\": {\"user\": \"fdw_user\", \"password\": \"foobarino\"}}}}')"
+
+    sql postgres "SELECT cartodb._CDB_Setup_FDW('test_fdw')"
+
+    sql postgres "SHOW server_version_num"
+    if [ "$RESULT" -gt 90499 ]
+    then
+        sql postgres "SELECT cartodb.CDB_Add_Remote_Table('test_fdw', 'foo')"
+        sql postgres "SELECT * from test_fdw.foo;"
+    else
+        echo "NOTICE: PostgreSQL version is less than 9.5 ($RESULT). Skipping CDB_Add_Remote_Table."
+        sql postgres "CREATE FOREIGN TABLE test_fdw.foo (a int) SERVER test_fdw OPTIONS (table_name 'foo', schema_name 'test_fdw')"
+    fi
+
+    sql postgres "SELECT n.nspname,
+  c.relname,
+  s.srvname FROM pg_catalog.pg_foreign_table ft
+  INNER JOIN pg_catalog.pg_class c ON c.oid = ft.ftrelid
+  INNER JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+  INNER JOIN pg_catalog.pg_foreign_server s ON s.oid = ft.ftserver
+ORDER BY 1, 2" should "test_fdw|cdb_tablemetadata|test_fdw
+test_fdw|foo|test_fdw"
+
+    sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo'::regclass) < NOW()" should 't'
+
+    sql postgres "SELECT a from test_fdw.foo LIMIT 1;" should 42
+
+    # Check function CDB_QueryTables_Updated_At
+    sql postgres 'CREATE TABLE local (b int);'
+    sql postgres 'INSERT INTO local (b) VALUES (43);'
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local'::regclass);"
+    local query='$query$ SELECT * FROM test_fdw.foo, local $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname;" should 'fdw_target|test_fdw|foo
+test_extension|public|local'
+    sql postgres "SELECT table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) order by updated_at;" should 'foo
+local'
+
+    # Check function CDB_Last_Updated_Time
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) > (now() - interval '1 minute')" should 't'
+
+    # Check we quote names on output as needed (as CDB_QueryTablesText does)
+    sql postgres 'CREATE TABLE "local-table-with-dashes" (c int)';
+    sql postgres 'INSERT INTO "local-table-with-dashes" (c) VALUES (44)';
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local-table-with-dashes'::regclass);"
+    query='$query$ SELECT * FROM test_fdw.foo, local, public."local-table-with-dashes" $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname, schema_name, table_name;" should 'fdw_target|test_fdw|foo
+test_extension|public|local
+test_extension|public|"local-table-with-dashes"'
+
+    # Check CDB_Last_Updated_Time supports quoted identifiers
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) > (now() - interval '1 minute')" should 't'
+
+    DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo2 FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
+
+    sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
+    DATABASE=fdw_target tear_down_database
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@

test/overviews/fixtures.sql

@@ -2240,3 +2240,10 @@ INSERT INTO polyg_t VALUES
 (3, 'C', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857)),
 (4, 'D', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857)),
 (5, 'E', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857));
+
+CREATE TABLE column_types_t
+AS SELECT cartodb_id,
+          the_geom,
+          the_geom_webmercator,
+          (CASE cartodb_id % 2 WHEN 0 THEN 'f' ELSE 't' END)::boolean AS is_odd
+FROM base_bare_t;