Merge pull request #208 from CartoDB/release_0_14_2

Release 0.14.2

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.13.1
+EXTVERSION = 0.14.2
 
 SED = sed
 
@@ -55,10 +55,13 @@ UPGRADABLE = \
   0.11.2 \
   0.11.3 \
   0.11.4 \
-	0.11.5 \
-	0.12.0 \
-	0.13.0 \
-	0.13.1 \
+  0.11.5 \
+  0.12.0 \
+  0.13.0 \
+  0.13.1 \
+  0.14.0 \
+  0.14.1 \
+  0.14.2 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -92,7 +95,8 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
     $(SED) -e 's/public\./cartodb./g' \
-        -e 's/:DATABASE_USERNAME/cdb_org_admin/g' >> $@
+        -e 's/:DATABASE_USERNAME/cdb_org_admin/g' \
+        -e "s/''public''/''cartodb''/g" >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 

NEWS.md

@@ -1,7 +1,21 @@
-0.13.0 (2016-02-01)
+0.14.2 (2016-03-15)
+-------------------
+* Support text `cartodb_id` columns in `_CDB_Has_Usable_Primary_ID` [#202](https://github.com/CartoDB/cartodb-postgresql/pull/202)
+
+0.14.1 (2016-03-07)
+-------------------
+* Fully qualify table names in cache cdb_invalidate_varnish calls [#198](https://github.com/CartoDB/cartodb-postgresql/issues/198)
+
+0.14.0 (2016-02-14)
+-------------------
+* Add CDB_ForeignTable.sql to support FDW's [#199](https://github.com/CartoDB/cartodb-postgresql/pull/199)
+
+0.13.1 (2016-02-01)
+-------------------
 * Fix migration fron unpackaged. [193](https://github.com/CartoDB/cartodb-postgresql/pull/193)
 
 0.13.0 (2016-01-29)
+-------------------
 * Add CDB_CreateOverviews, CDB_DropOverviews and CDB_Overviews for vector overviews support. [185](https://github.com/CartoDB/cartodb-postgresql/pull/185)
 * Convert some simple functions from plpgsql to sql. [188](https://github.com/CartoDB/cartodb-postgresql/pull/188)
 

scripts-available/CDB_CartodbfyTable.sql

@@ -470,6 +470,7 @@ $$ LANGUAGE 'plpgsql';
 -- If the table has both a usable key and usable geometry
 -- we can no-op on the table copy and just ensure that the 
 -- indexes and triggers are in place
+DROP FUNCTION IF EXISTS _CDB_Has_Usable_Primary_ID(reloid REGCLASS);
 CREATE OR REPLACE FUNCTION _CDB_Has_Usable_Primary_ID(reloid REGCLASS)
   RETURNS BOOLEAN
 AS $$
@@ -489,114 +490,98 @@ BEGIN
   -- Do we already have a properly named column?
   SELECT a.attname, i.indisprimary, i.indisunique, a.attnotnull, a.atttypid
   INTO rec
-  FROM pg_class c 
-  JOIN pg_attribute a ON a.attrelid = c.oid 
+  FROM pg_class c
+  JOIN pg_attribute a ON a.attrelid = c.oid
   JOIN pg_type t ON a.atttypid = t.oid
   LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
-  WHERE c.oid = reloid 
+  WHERE c.oid = reloid
   AND NOT a.attisdropped
   AND a.attname = const.pkey;
 
   -- Found something named right...
   IF FOUND THEN
-  
-    -- And it's an integer column...
-    IF rec.atttypid IN (20,21,23) THEN
-          
-      -- And it's a unique primary key! Done!
-      IF (rec.indisprimary OR rec.indisunique) AND rec.attnotnull THEN
-        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey);
-        RETURN true;
 
-      -- Check and see if the column values are unique and not null, 
-      -- if they are, we can use this column...
+    -- And it's a unique primary key! Done!
+    IF (rec.indisprimary OR rec.indisunique) AND rec.attnotnull THEN
+      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('found good ''%s''', const.pkey);
+      RETURN true;
+
+    -- Check and see if the column values are unique and not null,
+    -- if they are, we can use this column...
+    ELSE
+
+      -- Assume things are OK until proven otherwise...
+      useable_key := true;
+      BEGIN
+        sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey);
+        sql := sql || ', ' || Format('ADD CONSTRAINT %s_integer CHECK (%s::integer >=0);', const.pkey, const.pkey);
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql;
+        EXECUTE sql;
+        EXCEPTION
+        -- Failed unique check...
+        WHEN unique_violation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
+          useable_key := false;
+        -- Failed not null check...
+        WHEN not_null_violation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey);
+          useable_key := false;
+        -- Failed integer check...
+        WHEN invalid_text_representation THEN
+          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('invalid input syntax for integer %s', const.pkey);
+          useable_key := false;
+        -- Other fatal error
+        WHEN others THEN
+          PERFORM _CDB_Error(sql, Format('_CDB_Has_Usable_Primary_ID: %s', SQLERRM));
+      END;
+
+      -- Clean up test constraint
+      IF useable_key THEN
+        PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey));
+        PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_integer', reloid::text, const.pkey));
+
+      -- Move non-valid column out of the way
       ELSE
 
-        -- Assume things are OK until proven otherwise...
-        useable_key := true;
-      
-        BEGIN
-          sql := Format('ALTER TABLE %s ADD CONSTRAINT %s_pk PRIMARY KEY (%s)', reloid::text, const.pkey, const.pkey);
-          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', sql;
-          EXECUTE sql;
-          EXCEPTION      
-          -- Failed unique check...
-          WHEN unique_violation THEN
-            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s is not unique', const.pkey);
-            useable_key := false;
-          -- Failed not null check...
-          WHEN not_null_violation THEN
-            RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', Format('column %s contains nulls', const.pkey);
-            useable_key := false;
-          -- Other fatal error
-          WHEN others THEN
-            PERFORM _CDB_Error(sql, '_CDB_Has_Usable_Primary_ID');          
-        END;
-  
-        -- Clean up test constraint
-        IF useable_key THEN
-          PERFORM _CDB_SQL(Format('ALTER TABLE %s DROP CONSTRAINT %s_pk', reloid::text, const.pkey));
+        RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
+          Format('found non-valid ''%s''', const.pkey);
 
-        -- Move non-unique column out of the way
-        ELSE
-        
-          RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
-            Format('found non-unique ''%s'', renaming it', const.pkey);
-
-          PERFORM _CDB_SQL(
-            Format('ALTER TABLE %s RENAME COLUMN %s TO %I',
-              reloid::text, rec.attname,
-              cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)),
-            '_CDB_Has_Usable_Primary_ID');
-        
-        END IF;
-        
-        return useable_key;
+        PERFORM _CDB_Error(sql, Format('_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, %s', const.pkey));
 
       END IF;
-    
-    -- It's not an integer column, we have to rename it
-    ELSE
-  
-      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 
-        Format('found non-integer ''%s'', renaming it', const.pkey);
 
-      PERFORM _CDB_SQL(
-        Format('ALTER TABLE %s RENAME COLUMN %s TO %I',
-                reloid::text, rec.attname, cartodb._CDB_Unique_Column_Identifier(NULL, const.pkey, NULL, reloid)),
-                '_CDB_Has_Usable_Primary_ID');
-    
+      RETURN useable_key;
+
     END IF;
-    
+
   -- There's no column there named pkey
   ELSE
 
-    -- Is there another suitable primary key already?
+    -- Is there another integer suitable primary key already?
     SELECT a.attname
     INTO rec
     FROM pg_class c 
-    JOIN pg_attribute a ON a.attrelid = c.oid 
+    JOIN pg_attribute a ON a.attrelid = c.oid
     JOIN pg_type t ON a.atttypid = t.oid
     LEFT JOIN pg_index i ON c.oid = i.indrelid AND a.attnum = ANY(i.indkey)
     WHERE c.oid = reloid AND NOT a.attisdropped
     AND i.indisprimary AND i.indisunique AND a.attnotnull AND a.atttypid IN (20,21,23);
-  
+
     -- Yes! Ok, rename it.
     IF FOUND THEN
       PERFORM _CDB_SQL(Format('ALTER TABLE %s RENAME COLUMN %s TO %s', reloid::text, rec.attname, const.pkey),'_CDB_Has_Usable_Primary_ID');
       RETURN true;
     ELSE
-      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 
+      RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %',
         Format('found no useful column for ''%s''', const.pkey);
     END IF;
-  
+
   END IF;
-  
+
   RAISE DEBUG 'CDB(_CDB_Has_Usable_Primary_ID): %', 'function complete';
 
   -- Didn't find re-usable key, so return FALSE
   RETURN false;
-
 END;
 $$ LANGUAGE 'plpgsql';
 
@@ -825,10 +810,10 @@ DECLARE
   str TEXT;
   table_srid INTEGER;
   geom_srid INTEGER;
-  
+
   has_usable_primary_key BOOLEAN;
   has_usable_pk_sequence BOOLEAN;
-  
+
 BEGIN
 
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): %', 'entered function';
@@ -839,7 +824,7 @@ BEGIN
   -- Save the raw schema/table names for later
   SELECT n.nspname, c.relname, c.relname
   INTO STRICT relschema, relname, destname
-  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid 
+  FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
   WHERE c.oid = reloid;
 
   -- Default the destination to current schema if unspecified
@@ -890,7 +875,7 @@ BEGIN
 
   RAISE DEBUG 'CDB(_CDB_Rewrite_Table): has_usable_geoms %', gc.has_usable_geoms;
 
-  -- We can only avoid a rewrite if both the key and 
+  -- We can only avoid a rewrite if both the key and
   -- geometry are usable
 
   -- No table re-write is required, BUT a rename is required to
@@ -934,7 +919,7 @@ BEGIN
 
   -- Add cartodb ID!
   IF has_usable_primary_key THEN
-    sql := sql || const.pkey;
+    sql := sql || const.pkey || '::bigint ';
   ELSE
     sql := sql || 'nextval(''' || destseq || ''') AS ' || const.pkey;
   END IF;
@@ -1096,7 +1081,7 @@ BEGIN
 
   -- Run it!
   PERFORM _CDB_SQL(sql, '_CDB_Rewrite_Table');
-  
+
   -- Set up the primary key sequence
   -- If we copied the primary key from the original data, we need
   -- to set the sequence to the maximum value of that key

scripts-available/CDB_ForeignTable.sql

@@ -0,0 +1,199 @@
+---------------------------
+-- FDW MANAGEMENT FUNCTIONS
+--
+-- All the FDW settings are read from the `cdb_conf.fdws` entry json file.
+---------------------------
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text, config json)
+RETURNS void
+AS $$
+DECLARE
+  row record;
+  option record;
+  org_role text;
+BEGIN
+  -- This function tries to be as idempotent as possible, by not creating anything more than once
+  -- (not even using IF NOT EXIST to avoid throwing warnings)
+  IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
+    CREATE EXTENSION postgres_fdw;
+  END IF;
+  -- Create FDW first if it does not exist
+  IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_name)
+    THEN
+    EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_name);
+  END IF;
+
+  -- Set FDW settings
+  FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
+    LOOP
+      IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_name) SELECT * from a where options = row.key)
+        THEN
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_name, row.key, row.value);
+      ELSE
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_name, row.key, row.value);
+      END IF;
+    END LOOP;
+
+    -- Create user mappings
+    FOR row IN SELECT p.key, p.value from lateral json_each(config->'users') p LOOP
+        -- Check if entry on pg_user_mappings exists
+
+        IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_name AND usename = row.key ) THEN
+          EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', row.key, fdw_name);
+        END IF;
+
+    -- Update user mapping settings
+    FOR option IN SELECT o.key, o.value from lateral json_each_text(row.value) o LOOP
+        IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_name AND usename = row.key) SELECT * from a where options = option.key) THEN
+          EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', row.key, fdw_name, option.key, option.value);
+        ELSE
+          EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', row.key, fdw_name, option.key, option.value);
+        END IF;
+      END LOOP;
+    END LOOP;
+
+    -- Create schema if it does not exist.
+    IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_name) THEN
+      EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_name);
+    END IF;
+
+    -- Give the organization role usage permisions over the schema
+    SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() INTO org_role;
+    EXECUTE FORMAT ('GRANT USAGE ON SCHEMA %I TO %I', fdw_name, org_role);
+
+    -- Bring here the remote cdb_tablemetadata
+    IF NOT EXISTS ( SELECT * FROM PG_CLASS WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname=fdw_name) and relname='cdb_tablemetadata') THEN
+      EXECUTE FORMAT ('CREATE FOREIGN TABLE %I.cdb_tablemetadata (tabname text, updated_at timestamp with time zone) SERVER %I OPTIONS (table_name ''cdb_tablemetadata_text'', schema_name ''public'', updatable ''false'')', fdw_name, fdw_name);
+    END IF;
+    EXECUTE FORMAT ('GRANT SELECT ON %I.cdb_tablemetadata TO %I', fdw_name, org_role);
+
+END
+$$
+LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDWS()
+RETURNS VOID AS 
+$$
+DECLARE
+row record;
+BEGIN
+  FOR row IN SELECT p.key, p.value from lateral json_each(cartodb.CDB_Conf_GetConf('fdws')) p LOOP
+      EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING row.key, row.value;
+    END LOOP;
+  END
+$$
+LANGUAGE PLPGSQL;
+
+
+CREATE OR REPLACE FUNCTION cartodb._CDB_Setup_FDW(fdw_name text)
+  RETURNS void AS
+$BODY$
+DECLARE
+config json;
+BEGIN
+  SELECT p.value FROM LATERAL json_each(cartodb.CDB_Conf_GetConf('fdws')) p WHERE p.key = fdw_name INTO config;
+  EXECUTE 'SELECT cartodb._CDB_Setup_FDW($1, $2)' USING fdw_name, config;
+END
+$BODY$
+LANGUAGE plpgsql VOLATILE;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_Add_Remote_Table(source text, table_name text)
+  RETURNS void AS
+$$
+BEGIN
+  PERFORM cartodb._CDB_Setup_FDW(source);
+  EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', source, table_name, source, source);
+  --- Grant SELECT to publicuser
+  EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO publicuser;', source, table_name);
+END
+$$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_Get_Foreign_Updated_At(foreign_table regclass)
+  RETURNS timestamp with time zone AS
+$$
+DECLARE
+  remote_table_name text;
+  fdw_schema_name text;
+  time timestamp with time zone;
+BEGIN
+  -- This will turn a local foreign table (referenced as regclass) to its fully qualified text remote table reference.
+  WITH a AS (SELECT ftoptions FROM pg_foreign_table WHERE ftrelid=foreign_table LIMIT 1),
+    b as (SELECT (pg_options_to_table(ftoptions)).* FROM a)
+    SELECT FORMAT('%I.%I', (SELECT option_value FROM b WHERE option_name='schema_name'), (SELECT option_value FROM b WHERE option_name='table_name'))
+  INTO remote_table_name;
+
+  -- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table.
+  SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name;
+  EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
+  RETURN time;
+END
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid)
+RETURNS TEXT AS $$
+    SELECT option_value FROM pg_options_to_table((
+
+        SELECT fs.srvoptions
+        FROM pg_foreign_table ft
+        LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid
+        WHERE ft.ftrelid = reloid
+
+    )) WHERE option_name='dbname';
+$$ LANGUAGE SQL;
+
+
+-- Return a set of (dbname, schema_name, table_name, updated_at)
+-- It is aware of foreign tables
+-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
+-- Note: dbname is never quoted whereas schema and table names are when needed.
+CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text)
+RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz)
+AS $$
+    WITH query_tables AS (
+      SELECT unnest(CDB_QueryTablesText(query)) schema_table_name
+    ), query_tables_oid AS (
+      SELECT schema_table_name, schema_table_name::regclass::oid AS reloid
+      FROM query_tables
+    ),
+    fqtn AS (
+      SELECT
+        (CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid)
+              ELSE current_database()
+         END)::text AS dbname,
+         quote_ident(n.nspname::text) schema_name,
+         quote_ident(c.relname::text) table_name,
+         c.relkind,
+         query_tables_oid.reloid
+      FROM query_tables_oid, pg_catalog.pg_class c
+      LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+      WHERE c.oid = query_tables_oid.reloid
+    )
+    SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name,
+      (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
+            ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
+      END) AS updated_at
+    FROM fqtn;
+$$ LANGUAGE SQL;
+
+
+-- Return the last updated time of a set of tables
+-- It is aware of foreign tables
+-- It assumes the local (schema_name, table_name) map to the remote ones with the same name
+CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[])
+RETURNS timestamptz AS $$
+    WITH t AS (
+        SELECT unnest(tables) AS schema_table_name
+    ), t_oid AS (
+        SELECT (t.schema_table_name)::regclass::oid as reloid FROM t
+    ), t_updated_at AS (
+        SELECT
+            (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid)
+                  ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid)
+             END) AS updated_at
+        FROM t_oid
+        LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
+    ) SELECT max(updated_at) FROM t_updated_at;
+$$ LANGUAGE SQL;

scripts-available/CDB_TableMetadata.sql

@@ -5,6 +5,11 @@ CREATE TABLE IF NOT EXISTS
     updated_at timestamp with time zone not null default now()
   );
 
+CREATE OR REPLACE VIEW public.CDB_TableMetadata_Text AS
+       SELECT FORMAT('%I.%I', n.nspname::text, c.relname::text) tabname, updated_at
+       FROM public.CDB_TableMetadata, pg_catalog.pg_class c
+       LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid;
+
 -- No one can see this
 -- Updates are only possible trough the security definer trigger
 -- GRANT SELECT ON public.CDB_TableMetadata TO public;
@@ -61,9 +66,11 @@ CREATE OR REPLACE FUNCTION _CDB_TableMetadata_Updated()
 RETURNS trigger AS
 $$
 DECLARE
-  tabname TEXT;
+  tabname regclass;
   rec RECORD;
   found BOOL;
+  schema_name TEXT;
+  table_name TEXT;
 BEGIN
 
   IF TG_OP = 'UPDATE' or TG_OP = 'INSERT' THEN
@@ -98,9 +105,10 @@ BEGIN
                AND u.usesuper
              ORDER BY n.nspname
   LOOP
+    SELECT n.nspname, c.relname FROM pg_class c, pg_namespace n WHERE c.oid=tabname AND c.relnamespace = n.oid INTO schema_name, table_name;
     EXECUTE 'SELECT ' || quote_ident(rec.nspname) || '.'
             || quote_ident(rec.proname)
-            || '(' || quote_literal(tabname) || ')';
+            || '(' || quote_literal(quote_ident(schema_name) || '.' || quote_ident(table_name)) || ')';
     found := true;
     EXIT;
   END LOOP;
@@ -111,11 +119,11 @@ END;
 $$
 LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
 
-DROP TRIGGER IF EXISTS table_modified ON CDB_TableMetadata;
+DROP TRIGGER IF EXISTS table_modified ON public.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table
 --       oid (regclass) to its name
 CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
-ON CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
+ON public.CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
  _CDB_TableMetadata_Updated();
 
 

scripts-enabled/250-CDB_ForeignTable.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_ForeignTable.sql

test/CDB_CartodbfyTableTest.sql

@@ -1,5 +1,5 @@
 SET client_min_messages TO error;
-\set VERBOSITY default
+\set VERBOSITY terse
 
 CREATE OR REPLACE FUNCTION CDB_CartodbfyTableCheck(tabname regclass, label text)
 RETURNS text AS
@@ -124,13 +124,21 @@ END;
 $$
 LANGUAGE 'plpgsql';
 
--- table with single non-geometrical column
+-- check cartodbfytable idempotence
 CREATE TABLE t AS SELECT 1::int as a;
 SELECT CDB_CartodbfyTable('public', 't'); -- should fail
 SELECT CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
 SELECT CDB_CartodbfyTableCheck('t', 'single non-geometrical column');
 DROP TABLE t;
 
+-- table with single non-geometrical column
+CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
+SELECT * FROM t;
+DROP TABLE t;
+
 -- table with existing srid-unconstrained (but type-constrained) the_geom
 CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(0,0),4326)::geometry(point) as the_geom;
 SELECT CDB_CartodbfyTableCheck('t', 'srid-unconstrained the_geom');
@@ -164,19 +172,22 @@ SELECT CDB_CartodbfyTableCheck('t', 'trigger-protected the_geom');
 SELECT 'extent',ST_Extent(ST_SnapToGrid(the_geom,0.2)) FROM t;
 DROP TABLE t;
 
--- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
--- -- table with existing cartodb_id field of type text
--- CREATE TABLE t AS SELECT 10::text as cartodb_id;
--- SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
--- select cartodb_id/2 FROM t;
--- DROP TABLE t;
+-- table with existing cartodb_id field of type text
+CREATE TABLE t AS SELECT 10::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'text cartodb_id');
+select cartodb_id/2 FROM t;
+DROP TABLE t;
 
--- INFO: disabled because cartodbfy does not longer consider text columns for primary ID
--- -- table with existing cartodb_id field of type text not casting
--- CREATE TABLE t AS SELECT 'nan' as cartodb_id;
--- SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
--- select cartodb_id,_cartodb_id0 FROM t;
--- DROP TABLE t;
+-- table with existing cartodb_id field of type text not casting
+CREATE TABLE t AS SELECT 'nan'::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'uncasting text cartodb_id');
+DROP TABLE t;
+
+-- table with empty cartodb_id field of type text
+CREATE TABLE t AS SELECT null::text as cartodb_id;
+SELECT CDB_CartodbfyTableCheck('t', 'empty text cartodb_id');
+SELECT cartodb_id from t;
+DROP TABLE t;
 
 -- table with existing cartodb_id field of type int4 not sequenced
 CREATE TABLE t AS SELECT 1::int4 as cartodb_id;
@@ -291,7 +302,7 @@ INSERT INTO test VALUES
   (NULL),
   (3);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with null cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 -- Table with non unique cartodb_id
@@ -303,7 +314,7 @@ INSERT INTO test VALUES
   (2),
   (2);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 -- Table with non unique and null cartodb_id
@@ -316,7 +327,7 @@ INSERT INTO test VALUES
   (NULL),
   (2);
 SELECT CDB_CartodbfyTableCheck('test', 'Table with non unique and null cartodb_id #148');
-SELECT cartodb_id, cartodb_id_0 from test;
+SELECT cartodb_id from test;
 DROP TABLE test;
 
 CREATE TABLE test (

test/CDB_CartodbfyTableTest_expect

@@ -2,12 +2,16 @@ SET
 CREATE FUNCTION
 SELECT 1
 ERROR:  Please set user quota before cartodbfying tables.
-CONTEXT:  SQL statement "SELECT cartodb._CDB_check_prerequisites(destschema, reloid)"
-PL/pgSQL function cdb_cartodbfytable(text,regclass) line 21 at PERFORM
 0
 single non-geometrical column cartodbfied fine
 DROP TABLE
 SELECT 1
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+check function idempotence cartodbfied fine
+1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
+DROP TABLE
+SELECT 1
 srid-unconstrained the_geom cartodbfied fine
 DROP TABLE
 SELECT 2
@@ -26,6 +30,17 @@ SELECT 1
 CREATE TRIGGER
 trigger-protected the_geom cartodbfied fine
 extent|BOX(1 1,2 2)
+DROP TABLE
+SELECT 1
+text cartodb_id cartodbfied fine
+5
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+DROP TABLE
+SELECT 1
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE t ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+
 DROP TABLE
 SELECT 1
 unsequenced cartodb_id cartodbfied fine
@@ -77,30 +92,30 @@ Table with both the_geom and wkb_geometry #141 cartodbfied fine
 DROP TABLE
 CREATE TABLE
 INSERT 0 1
-Many colliding columns #141 cartodbfied fine
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: multiple primary keys for table "many_colliding_columns" are not allowed): ALTER TABLE many_colliding_columns ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
 DROP TABLE
 CREATE TABLE
 INSERT 0 4
-Table with null cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|
-4|3
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+3
 DROP TABLE
 CREATE TABLE
 INSERT 0 3
-Table with non unique cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|2
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+2
 DROP TABLE
 CREATE TABLE
 INSERT 0 4
-Table with non unique and null cartodb_id #148 cartodbfied fine
-1|1
-2|2
-3|
-4|2
+ERROR:  CDB(_CDB_Has_Usable_Primary_ID: Error: invalid cartodb_id, cartodb_id): ALTER TABLE test ADD CONSTRAINT cartodb_id_pk PRIMARY KEY (cartodb_id), ADD CONSTRAINT cartodb_id_integer CHECK (cartodb_id::integer >=0);
+1
+2
+
+2
 DROP TABLE
 CREATE TABLE
 CREATE INDEX

test/extension/test.sh

@@ -172,8 +172,7 @@ function drop_raster_table() {
     sql ${ROLE} "DROP TABLE ${ROLE}.${TABLENAME};"
 }
 
-
-function setup() {
+function setup_database() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
@@ -184,7 +183,10 @@ function setup() {
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
     # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
     ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
+}
 
+function setup() {
+    setup_database
 
     log_info "############################# SETUP #############################"
     create_role_and_schema cdb_testmember_1
@@ -199,6 +201,10 @@ function setup() {
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
 }
 
+
+function tear_down_database() {
+    ${CMD} -c "DROP DATABASE ${DATABASE}"
+}
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -219,9 +225,10 @@ function tear_down() {
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
 
-    ${CMD} -c "DROP DATABASE ${DATABASE}"
+    tear_down_database
 }
 
+
 function run_tests() {
     local FAILED_TESTS=()
 
@@ -343,6 +350,41 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
     sql postgres "REVOKE ALL ON CDB_TableMetadata FROM cdb_testmember_1;"
 }
 
+function test_cdb_tablemetadatatouch_fully_qualifies_names() {
+    sql postgres "CREATE TABLE touch_invalidations (table_name text);"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+
+    #default schema
+    sql "CREATE TABLE touch_example (a int);"
+    sql postgres "SELECT CDB_TableMetadataTouch('touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "public.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+    sql postgres "DROP TABLE touch_example"
+
+    #setup different schema
+    sql postgres "CREATE SCHEMA test_schema;"
+    sql postgres "CREATE TABLE test_schema.touch_example (a int);"
+
+    #different schema outside search_path
+    sql postgres "SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #different schema in default search_path
+    sql postgres "SET search_path=test_schema,public,cartodb; SELECT CDB_TableMetadataTouch('test_schema.touch_example');"
+    sql postgres "SELECT table_name FROM touch_invalidations" should "test_schema.touch_example"
+    sql postgres "TRUNCATE TABLE touch_invalidations"
+
+    #teardown different schema
+    sql postgres 'DROP TABLE test_schema.touch_example;'
+    sql postgres 'DROP SCHEMA test_schema;'
+
+
+
+    sql postgres 'DROP FUNCTION cartodb.cdb_invalidate_varnish(table_name text);'
+    sql postgres 'DROP TABLE touch_invalidations'
+}
+
 function test_cdb_column_names() {
     sql cdb_testmember_1 'CREATE TABLE cdb_testmember_1.table_cnames(c int, a int, r int, t int, o int);'
     sql cdb_testmember_2 'CREATE TABLE cdb_testmember_2.table_cnames(d int, b int);'
@@ -429,6 +471,98 @@ function test_cdb_querytables_happy_cases() {
     sql postgres 'DROP SCHEMA foo;'
 }
 
+function test_foreign_tables() {
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_TableMetadata.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_ForeignTable.sql
+
+
+    DATABASE=fdw_target setup_database
+    ${CMD} -d fdw_target -f scripts-available/CDB_QueryStatements.sql
+    ${CMD} -d fdw_target -f scripts-available/CDB_QueryTables.sql
+    ${CMD} -d fdw_target -f scripts-available/CDB_TableMetadata.sql
+
+    DATABASE=fdw_target sql postgres 'CREATE SCHEMA test_fdw;'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo (a) values (42);'
+    DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo2 (a int);'
+    DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo2 (a) values (42);'
+    DATABASE=fdw_target sql postgres "CREATE USER fdw_user WITH PASSWORD 'foobarino';"
+    DATABASE=fdw_target sql postgres 'GRANT USAGE ON SCHEMA test_fdw TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON TABLE test_fdw.foo2 TO fdw_user;'
+    DATABASE=fdw_target sql postgres 'GRANT SELECT ON cdb_tablemetadata_text TO fdw_user;'
+
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
+    DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('fdws', '{\"test_fdw\": {\"server\": {\"host\": \"localhost\", \"dbname\": \"fdw_target\"},
+                                           \"users\": {\"public\": {\"user\": \"fdw_user\", \"password\": \"foobarino\"}}}}')"
+
+    sql postgres "SELECT cartodb._CDB_Setup_FDW('test_fdw')"
+
+    sql postgres "SHOW server_version_num"
+    if [ "$RESULT" -gt 90499 ]
+    then
+        sql postgres "SELECT cartodb.CDB_Add_Remote_Table('test_fdw', 'foo')"
+        sql postgres "SELECT * from test_fdw.foo;"
+    else
+        echo "NOTICE: PostgreSQL version is less than 9.5 ($RESULT). Skipping CDB_Add_Remote_Table."
+        sql postgres "CREATE FOREIGN TABLE test_fdw.foo (a int) SERVER test_fdw OPTIONS (table_name 'foo', schema_name 'test_fdw')"
+    fi
+
+    sql postgres "SELECT n.nspname,
+  c.relname,
+  s.srvname FROM pg_catalog.pg_foreign_table ft
+  INNER JOIN pg_catalog.pg_class c ON c.oid = ft.ftrelid
+  INNER JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+  INNER JOIN pg_catalog.pg_foreign_server s ON s.oid = ft.ftserver
+ORDER BY 1, 2" should "test_fdw|cdb_tablemetadata|test_fdw
+test_fdw|foo|test_fdw"
+
+    sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo'::regclass) < NOW()" should 't'
+
+    sql postgres "SELECT a from test_fdw.foo LIMIT 1;" should 42
+
+    # Check function CDB_QueryTables_Updated_At
+    sql postgres 'CREATE TABLE local (b int);'
+    sql postgres 'INSERT INTO local (b) VALUES (43);'
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local'::regclass);"
+    local query='$query$ SELECT * FROM test_fdw.foo, local $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname;" should 'fdw_target|test_fdw|foo
+test_extension|public|local'
+    sql postgres "SELECT table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) order by updated_at;" should 'foo
+local'
+
+    # Check function CDB_Last_Updated_Time
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) > (now() - interval '1 minute')" should 't'
+
+    # Check we quote names on output as needed (as CDB_QueryTablesText does)
+    sql postgres 'CREATE TABLE "local-table-with-dashes" (c int)';
+    sql postgres 'INSERT INTO "local-table-with-dashes" (c) VALUES (44)';
+    sql postgres "SELECT cdb_tablemetadatatouch('public.local-table-with-dashes'::regclass);"
+    query='$query$ SELECT * FROM test_fdw.foo, local, public."local-table-with-dashes" $query$::text'
+    sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname, schema_name, table_name;" should 'fdw_target|test_fdw|foo
+test_extension|public|local
+test_extension|public|"local-table-with-dashes"'
+
+    # Check CDB_Last_Updated_Time supports quoted identifiers
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) < now()" should 't'
+    sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) > (now() - interval '1 minute')" should 't'
+
+    DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo2 FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
+    DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
+
+    sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
+    DATABASE=fdw_target tear_down_database
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@