Release 0.31.0

Sec definer

.travis.yml

@@ -24,7 +24,7 @@ before_install:
   - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
   - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
   - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
-  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password
   - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
   - sudo pip install redis==2.4.9
 script:

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.27.0
+EXTVERSION = 0.31.0
 
 SED = sed
 AWK = awk
@@ -97,6 +97,13 @@ UPGRADABLE = \
   0.26.0 \
   0.26.1 \
   0.27.0 \
+  0.27.1 \
+  0.27.2 \
+  0.28.0 \
+  0.28.1 \
+  0.29.0 \
+  0.30.0 \
+  0.31.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -126,6 +133,8 @@ PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+
 $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
@@ -167,7 +176,11 @@ legacy_regress: $(REGRESS_OLD) Makefile
 			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \
 		exp=expected/test/$${tn}.out; \
 		echo '\set ECHO none' > $${exp}; \
-		cat test/$${tn}_expect >> $${exp}; \
+		if [[ -f "test/$${tn}_expect.pg$(PG_VERSION)" ]]; then \
+			cat test/$${tn}_expect.pg$(PG_VERSION) >> $${exp}; \
+		else \
+			cat test/$${tn}_expect >> $${exp}; \
+		fi \
 	done
 
 test_organization:

NEWS.md

@@ -1,3 +1,29 @@
+0.31.0 (2019-10-08)
+* Ghost tables: Add missing tags (#370)
+* Set search_path in security definer functions.
+
+0.30.0 (2019-07-17)
+* Added new admin functions to connect CARTO with user FDW's (#369)
+
+0.29.0 (2019-07-15)
+* Added new function CDB_OAuth:
+  * Install event trigger to check for table/view/sequence/function creation
+  * Reassign the ownership of new objects to a defined role in the cdb_conf
+* Changed MakeFile to support different expects for differents PG versions
+
+0.28.1 (2019-07-04)
+* Avoid temporary tables creation in CDB_SyncTable (#366)
+* Make CDB_Get_Foreign_Updated_At robust to missing CDB_TableMetadata (#362)
+
+0.28.0 (2019-07-01)
+* New function CDB_SyncTable (#355)
+
+0.27.2 (2019-06-21)
+* Improvements and fixes in Ghost tables functions (#360)
+
+0.27.1 (2019-06-03)
+* Add some qualifications that were left in the previous release.
+
 0.27.0 (2019-06-03)
 * Fully qualify function calls
 * Several improvements to bash tests.

carto-package.json

@@ -2,7 +2,7 @@
     "name": "carto_postgresql_ext",
     "current_version": {
         "requires": {
-            "postgresql": ">=10.0",
+            "postgresql": ">=10.0.0",
             "postgis": ">=2.4.0.0"
         },
         "works_with": {

doc/CDB_SyncTable.md

@@ -0,0 +1,56 @@
+Synchronize two tables. This function will synchronize a *destination* table with a *source* table.
+The idea is that the *destination* is a replica of *source* and *source* has been subject to
+modifications that are to be applied to *destination*.
+
+This will be achieved by deleting the rows in the destination not present
+in the source, inserting rows of the source not in the destination and updating modified rows.
+If the destination table does not exist it will be created and all the rows of the source inserted into it.
+
+Both tables must have a consistent `cartodb_id` primary key column which will be used to match
+the source and destination rows.
+
+Note that both tables do not necessarily become identical after the synchronization, since additional columns
+may have been added to the destination; those columns will not be altered by the synchronization.
+
+In addition some source columns may be skipped by listing them in the optional last argument; such columns
+will not be updated in the destination, so if they are present in it their values won't be altered.
+
+
+#### Using the function
+
+Import some data using COPY FROM into a temporary table, then synchronize a table with the data and
+finally delete the temporary table. This could be used import and update some data periodically while
+allowing to add columns to the data that will be preserved across updates.
+
+```sql
+CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision, rank int);
+COPY tmp_pois FROM '/tmp/pois.csv';
+SELECT CDB_SyncTable('tmp_pois', 'public', 'pois');
+DROP TABLE tmp_pois;
+```
+
+Now we could perform some changes to the `pois` to maintain our own ranking:
+
+```sql
+UPDATE pois SET rank = random()*4 + 1;
+```
+
+Then, if the source were updated at `/tmp/pois.csv` we could synchronize with it while preserving our `rank` values with:
+
+```sql
+CREATE tmp_pois(cartodb_id int, name text, type text, longitude double precision, latitude double precision, rank int);
+COPY tmp_pois FROM '/tmp/pois.csv';
+SELECT CDB_SyncTable('tmp_pois', 'public', 'pois', '{rank}');
+DROP TABLE tmp_pois;
+```
+
+#### Arguments
+
+```
+CDB_SyncTable(src_table, dst_schema, dst_table, skip_cols)
+```
+
+* **src_table** REGCLASS the source data for the synchronization
+* **dst_scgena** REGNAMESPACE the destination schema
+* **dst_table** NAME the destination table to be updated
+* **skip_cols** NAME[] an array of column names, empty by default, which will be skipped

scripts-available/CDB_AnalysisCheck.sql

@@ -6,8 +6,11 @@ $$
 BEGIN
   RETURN @extschema@.CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
 END;
-$$
-LANGUAGE 'plpgsql' STABLE PARALLEL SAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 
 -- Get the factor (fraction of the quota) for Camshaft cached analysis tables

scripts-available/CDB_CartodbfyTable.sql

@@ -1071,7 +1071,7 @@ BEGIN
   -- by selecting their names into an array and
   -- joining the array with a comma
   SELECT
-    ',' || array_to_string(array_agg(Format('%I',a.attname)),',') AS column_name_sql,
+    ',' || array_to_string(array_agg(Format('%I',a.attname) ORDER BY a.attnum),',') AS column_name_sql,
     Count(*) AS count
   INTO rec
   FROM pg_class c

scripts-available/CDB_EstimateRowCount.sql

@@ -12,7 +12,12 @@ BEGIN
     EXECUTE Format('ANALYZE %s;', reloid);
   END IF;
 END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    VOLATILE
+    STRICT
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Return a row count estimate of the result of a query using statistics
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EstimateRowCount(query text)

scripts-available/CDB_ForeignTable.sql

@@ -125,13 +125,190 @@ BEGIN
 
   -- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table.
   SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name;
-  EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
+  BEGIN
+    EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time;
+  EXCEPTION
+    WHEN undefined_table THEN
+      -- If you add a GET STACKED DIAGNOSTICS text_var = RETURNED_SQLSTATE
+      -- you get a code 42P01 which corresponds to undefined_table
+      RAISE NOTICE 'CDB_Get_Foreign_Updated_At: could not find %.cdb_tablemetadata while checking % updated_at, returning NULL timestamp', fdw_schema_name, foreign_table;
+  END;
   RETURN time;
 END
 $$
 LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
 
 
+-- Produce a valid DB name for objects created for the user FDW's
+CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME)
+RETURNS NAME AS $$
+  -- Note on input we use %s and on output we use %I, in order to
+  -- avoid double escaping
+  SELECT format('cdb_fdw_%s', fdw_input_name)::NAME;
+$$
+LANGUAGE sql IMMUTABLE PARALLEL SAFE;
+
+-- A function to set up a user-defined foreign data server
+-- It does not read from CDB_Conf.
+-- Only superuser roles can invoke it successfully
+--
+-- Sample call:
+-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{
+--    "server": {
+--      "extensions": "postgis",
+--      "dbname": "testdb",
+--      "host": "myhostname.us-east-2.rds.amazonaws.com",
+--      "port": "5432"
+--    },
+--    "user_mapping": {
+--      "user": "fdw_user",
+--      "password": "secret"
+--    }
+-- }');
+--
+-- Underneath it will:
+--   * Set up postgresql_fdw
+--   * Create a server with the name 'cdb_fdw_amazon'
+--   * Create a role called 'cdb_fdw_amazon' to manage access
+--   * Create a user mapping with that role 'cdb_fdw_amazon'
+--   * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign
+--     tables over there
+--
+-- It is the responsibility of the superuser to grant that role to either:
+--   * Nobody
+--   * Specific roles: GRANT amazon TO role_name;
+--   * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon');
+--   * The publicuser: GRANT cdb_fdw_amazon TO publicuser;
+CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json)
+RETURNS void AS $$
+DECLARE
+  row record;
+  option record;
+  fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+BEGIN
+  -- TODO: refactor with original function
+  -- This function tries to be as idempotent as possible, by not creating anything more than once
+  -- (not even using IF NOT EXIST to avoid throwing warnings)
+  IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
+    CREATE EXTENSION postgres_fdw;
+    RAISE NOTICE 'Created postgres_fdw EXTENSION';
+  END IF;
+  -- Create FDW first if it does not exist
+  IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name)
+    THEN
+    EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name);
+    RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name;
+  END IF;
+
+  -- Set FDW settings
+  FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
+    LOOP
+      IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key)
+        THEN
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value);
+      ELSE
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value);
+      END IF;
+    END LOOP;
+
+    -- Create specific role for this
+    IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN
+       EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name);
+       RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name;
+    END IF;
+
+    -- Transfer ownership of the server to the fdw role
+    EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
+
+    -- Create user mapping
+    -- NOTE: we use a PUBLIC user mapping but control access to the SERVER
+    -- so that we don't need to create a mapping for every user nor store credentials elsewhere
+    IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN
+        EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name);
+        RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name;
+    END IF;
+
+    -- Update user mapping settings
+    FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP
+        IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value);
+        ELSE
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value);
+        END IF;
+    END LOOP;
+
+    -- Grant usage on the wrapper and server to the fdw role
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name);
+    RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name;
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name);
+    RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name;
+
+    -- Create schema if it does not exist.
+    IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN
+      EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name);
+      RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name;
+    END IF;
+
+    -- Give the fdw role ownership over the schema
+    EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
+    RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name;
+
+    -- TODO: Bring here the remote cdb_tablemetadata
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
+-- A function to drop a user-defined foreign server and all related objects
+-- It does not read from CDB_Conf
+-- It must be executed with a superuser role to succeed
+--
+-- Sample call:
+--   SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon')
+--
+-- Note: if there's any dependent object (i.e. foreign table) this call will fail
+CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false)
+RETURNS void AS $$
+DECLARE
+    fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+    cascade_clause NAME;
+BEGIN
+    CASE force
+        WHEN true THEN
+            cascade_clause := 'CASCADE';
+        ELSE
+            cascade_clause := 'RESTRICT';
+    END CASE;
+
+    EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Dropped schema %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name);
+    RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Dropped foreign server %', fdw_objects_name;
+    EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name);
+    RAISE NOTICE 'Dropped role %', fdw_objects_name;
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
+-- Set up a user foreign table
+-- E.g:
+--   SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable');
+--   SELECT * FROM amazon.my_table;
+CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME)
+RETURNS void AS $$
+DECLARE
+    fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+BEGIN
+  EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name);
+  --- Grant SELECT to fdw role
+  EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name);
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
 CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid)
 RETURNS TEXT AS $$
     SELECT option_value FROM pg_options_to_table((

scripts-available/CDB_GhostTables.sql

@@ -63,7 +63,11 @@ AS $$
     PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
     RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Trigger function to call CDB_LinkGhostTables()
 CREATE OR REPLACE FUNCTION @extschema@._CDB_LinkGhostTablesTrigger()
@@ -76,16 +80,24 @@ AS $$
     PERFORM @extschema@.CDB_LinkGhostTables(ddl_tag);
     RETURN NULL;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Event trigger to save the current transaction in @extschema@.cdb_ddl_execution
 CREATE OR REPLACE FUNCTION @extschema@.CDB_SaveDDLTransaction()
 RETURNS event_trigger
 AS $$
   BEGIN
-    INSERT INTO @extschema@.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING;
+    INSERT INTO @extschema@.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT ON CONSTRAINT cdb_ddl_execution_pkey DO NOTHING;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Creates the trigger on DDL events to link ghost tables
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableGhostTablesTrigger()
@@ -96,7 +108,7 @@ AS $$
     DROP TRIGGER IF EXISTS check_ddl_update ON @extschema@.cdb_ddl_execution;
 
     -- Table to store the transaction id from DDL events to avoid multiple executions
-    CREATE TABLE IF NOT EXISTS @extschema@.cdb_ddl_execution(txid integer PRIMARY KEY, tag text);
+    CREATE TABLE IF NOT EXISTS @extschema@.cdb_ddl_execution(txid bigint PRIMARY KEY, tag text);
 
     CREATE CONSTRAINT TRIGGER check_ddl_update
     AFTER INSERT ON @extschema@.cdb_ddl_execution
@@ -106,7 +118,33 @@ AS $$
 
     CREATE EVENT TRIGGER link_ghost_tables
     ON ddl_command_end
-    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW')
+    WHEN TAG IN ('CREATE TABLE',
+                'SELECT INTO',
+                'DROP TABLE',
+                'ALTER TABLE',
+
+                'CREATE TRIGGER',
+                'DROP TRIGGER',
+                'ALTER TRIGGER',
+
+                'CREATE VIEW',
+                'DROP VIEW',
+                'ALTER VIEW',
+
+                'CREATE FOREIGN TABLE',
+                'ALTER FOREIGN TABLE',
+                'DROP FOREIGN TABLE',
+
+                'ALTER MATERIALIZED VIEW',
+                'CREATE MATERIALIZED VIEW',
+                'DROP MATERIALIZED VIEW',
+
+                'IMPORT FOREIGN SCHEMA',
+
+                'DROP EXTENSION',
+                'DROP SCHEMA',
+                'DROP SERVER',
+                'DROP TYPE')
     EXECUTE PROCEDURE @extschema@.CDB_SaveDDLTransaction();
   END;
 $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Groups_API.sql

@@ -22,7 +22,11 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text)
@@ -35,7 +39,11 @@ $$
 
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
@@ -48,7 +56,11 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[])
@@ -61,7 +73,11 @@ $$
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
@@ -74,7 +90,11 @@ $$
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -95,7 +115,11 @@ $$
     body = '{ "access": "%s" }' % access
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -115,7 +139,11 @@ $$
     url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN

scripts-available/CDB_OAuth.sql

@@ -0,0 +1,56 @@
+-- Function that reassign the owner of a table to their ownership_role
+CREATE OR REPLACE FUNCTION @extschema@.CDB_OAuthReassignTableOwnerOnCreation()
+  RETURNS event_trigger
+  AS $$
+DECLARE
+    obj record;
+    owner_role text;
+    creator_role text;
+BEGIN
+    FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
+    LOOP
+      RAISE DEBUG '% ddl object: % % % %',
+                  tg_tag,
+                  obj.command_tag,
+                  obj.object_type,
+                  obj.schema_name,
+                  obj.object_identity;
+      SELECT rolname FROM pg_class JOIN pg_roles ON relowner = pg_roles.oid WHERE pg_class.oid = obj.objid INTO creator_role;
+      SELECT value->>'ownership_role_name' from @extschema@.CDB_Conf_GetConf('api_keys_' || quote_ident(creator_role)) value INTO owner_role;
+      IF owner_role IS NULL OR owner_role = '' THEN
+        CONTINUE;
+      ELSE
+        EXECUTE 'ALTER ' || obj.object_type || ' ' || obj.object_identity || ' OWNER TO ' || quote_ident(owner_role);
+        EXECUTE 'GRANT ALL ON ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        RAISE DEBUG 'Changing ownership from % to %', creator_role, owner_role;
+      END IF;
+    END LOOP;
+END;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
+
+-- Creates the trigger on DDL events in order to reassign the owner
+CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableOAuthReassignTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
+
+    CREATE EVENT TRIGGER oauth_reassign_tables_trigger
+    ON ddl_command_end
+    WHEN TAG IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'CREATE VIEW', 'CREATE FOREIGN TABLE', 'CREATE MATERIALIZED VIEW', 'CREATE SEQUENCE', 'CREATE FUNCTION')
+    EXECUTE PROCEDURE @extschema@.CDB_OAuthReassignTableOwnerOnCreation();
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+-- Deletes the trigger on DDL events in order to reassign the owner
+CREATE OR REPLACE FUNCTION @extschema@.CDB_DisableOAuthReassignTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Organizations.sql

@@ -169,3 +169,30 @@ BEGIN
     EXECUTE 'SELECT @extschema@.CDB_Organization_Remove_Access_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || @extschema@.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
+
+
+--------------------------------------------------------------------------------
+-- Role management
+--------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name)
+RETURNS VOID AS $$
+DECLARE
+    org_role TEXT;
+BEGIN
+    org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
+    EXECUTE format('GRANT %I TO %I', role_name, org_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
+
+
+CREATE OR REPLACE
+FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name)
+RETURNS VOID AS $$
+DECLARE
+    org_role TEXT;
+BEGIN
+    org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
+    EXECUTE format('REVOKE %I FROM %I', role_name, org_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Overviews.sql

@@ -66,7 +66,7 @@ AS $$
     FROM
       @extschema@._CDB_UserTablesInSchema(), unnest(tables) base_table
     WHERE
-      schema_name = _cdb_schema_name(base_table)
+      schema_name = @extschema@._cdb_schema_name(base_table)
       AND @extschema@._CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), table_name)
     ORDER BY base_table, z;
 $$ LANGUAGE SQL STABLE PARALLEL SAFE;
@@ -318,7 +318,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 -- This function is declared SECURITY DEFINER so it executes with the privileges
 -- of the function creator to have a chance to alter the privileges of the
 -- overview table to match those of the dataset. It will only perform any change
--- if the overview table belgons to the same scheme as the dataset and it
+-- if the overview table belongs to the same scheme as the dataset and it
 -- matches the scheme naming for overview tables.
 CREATE OR REPLACE FUNCTION @extschema@._CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
 RETURNS VOID
@@ -362,7 +362,11 @@ AS $$
       -- it should be done here (CDB_Overviews would consume such metadata)
     END IF;
   END
-$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE PLPGSQL
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Dataset attributes (column names other than the
 -- CartoDB primary key and geometry columns) which should be aggregated

scripts-available/CDB_SyncTable.sql

@@ -0,0 +1,167 @@
+/*
+   Gets the column names of a given table.
+
+   Sample usage:
+
+     SELECT @extschema@._CDB_GetColumns('public.films');
+*/
+CREATE OR REPLACE FUNCTION @extschema@._CDB_GetColumns(src_table REGCLASS)
+RETURNS SETOF NAME
+AS $$
+  SELECT
+    a.attname as "colname"
+  FROM
+    pg_catalog.pg_attribute a
+  WHERE
+    a.attnum > 0
+      AND NOT a.attisdropped
+      AND a.attrelid = (
+        SELECT c.oid
+          FROM pg_catalog.pg_class c
+          LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          WHERE c.oid = src_table::oid
+            AND pg_catalog.pg_table_is_visible(c.oid)
+      )
+  ORDER BY a.attnum;
+$$ LANGUAGE sql STABLE PARALLEL UNSAFE;
+
+
+/*
+    Given an array of quoted column names, it generates an UPDATE SET
+    clause with the following form:
+
+        the_geom = changed.the_geom,
+        id = changed.id,
+        elevation = changed.elevation
+
+    Example of usage:
+
+       SELECT @extschema@.__CDB_GetUpdateSetClause('{the_geom, id, elevation}', 'changed');
+*/
+CREATE OR REPLACE FUNCTION @extschema@.__CDB_GetUpdateSetClause(colnames TEXT[], update_source TEXT)
+RETURNS TEXT
+AS $$
+DECLARE
+  set_clause_list TEXT[];
+  col TEXT;
+BEGIN
+  FOREACH col IN ARRAY colnames
+  LOOP
+    set_clause_list := array_append(set_clause_list, format('%1$s = %2$s.%1$s', col, update_source));
+  END lOOP;
+  RETURN array_to_string(set_clause_list, ', ');
+END;
+$$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
+
+
+/*
+    Given a prefix, generate a safe unique NAME for a temp table.
+
+    Example of usage:
+
+       SELECT @extschema@.__CDB_GenerateUniqueName('src_sync'); --> src_sync_718794_120106
+
+*/
+CREATE OR REPLACE FUNCTION @extschema@.__CDB_GenerateUniqueName(prefix TEXT)
+RETURNS NAME
+AS $$
+  SELECT format('%s_%s_%s', prefix, txid_current(), (random()*1000000)::int)::NAME;
+$$ LANGUAGE sql VOLATILE PARALLEL UNSAFE;
+
+/*
+    Given a table name and an array of column names,
+    return array of column names qualified with the table name and quoted when necessary
+    tablename and colnames should be properly quoted, and for this reason the type NAME is not
+    used for them (with quotes they could exceed the maximum identifier length)
+
+    Example of usage:
+
+       SELECT @extschema@.__CDB_QualifyColumns('t', ARRAY['a','"b-1"']); --> ARRAY['t.a','t."b-1"']
+
+*/
+CREATE OR REPLACE FUNCTION @extschema@.__CDB_QualifyColumns(tablename NAME, colnames NAME[]) RETURNS TEXT[] AS
+$$
+    SELECT array_agg(tablename || '.' || _colname) from unnest(colnames) _colname;
+$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE;
+
+/*
+   A Table Syncer
+
+   Assumptions:
+     - Both tables contain a consistent cartodb_id column
+     - Destination table has all columns of the source or does not exist
+
+   Sample usage:
+
+     SELECT CDB_SyncTable('radar_stations', 'public', 'syncdest');
+     SELECT CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}');
+
+*/
+CREATE OR REPLACE FUNCTION @extschema@.CDB_SyncTable(src_table REGCLASS, dst_schema REGNAMESPACE, dst_table NAME, skip_cols NAME[] = '{}')
+RETURNS void
+AS $$
+DECLARE
+  fq_dest_table TEXT;
+
+  colnames TEXT[];
+  dst_colnames TEXT;
+  src_colnames TEXT;
+
+  update_set_clause TEXT;
+
+  num_rows BIGINT;
+  err_context text;
+
+  t timestamptz;
+BEGIN
+  -- If the destination table does not exist, just copy the source table
+  fq_dest_table := format('%s.%I', dst_schema, dst_table);
+  EXECUTE format('CREATE TABLE IF NOT EXISTS %s as TABLE %s', fq_dest_table, src_table);
+  GET DIAGNOSTICS num_rows = ROW_COUNT;
+  IF num_rows > 0 THEN
+    RAISE NOTICE 'INSERTED % row(s)', num_rows;
+    RETURN;
+  END IF;
+
+  skip_cols := skip_cols || '{cartodb_id}';
+
+  -- Get the list of columns from the source table, excluding skip_cols
+  SELECT ARRAY(SELECT quote_ident(c) FROM @extschema@._CDB_GetColumns(src_table) as c EXCEPT SELECT unnest(skip_cols)) INTO colnames;
+
+  -- Deal with deleted rows: ids in dest but not in source
+  t := clock_timestamp();
+  EXECUTE format(
+    'DELETE FROM %1$s _dst WHERE NOT EXISTS (SELECT * FROM %2$s _src WHERE _src.cartodb_id=_dst.cartodb_id)',
+    fq_dest_table, src_table);
+  GET DIAGNOSTICS num_rows = ROW_COUNT;
+  RAISE NOTICE 'DELETED % row(s)', num_rows;
+  RAISE DEBUG 'DELETE time (s): %', clock_timestamp() - t;
+
+  -- Deal with inserted rows: ids in source but not in dest
+  t := clock_timestamp();
+  EXECUTE format('
+      INSERT INTO %1$s(cartodb_id, %2$s)
+      SELECT cartodb_id, %2$s FROM %3$s _src WHERE NOT EXISTS (SELECT * FROM %1$s _dst WHERE _src.cartodb_id=_dst.cartodb_id)
+  ', fq_dest_table, array_to_string(colnames, ','), src_table);
+  GET DIAGNOSTICS num_rows = ROW_COUNT;
+  RAISE NOTICE 'INSERTED % row(s)', num_rows;
+  RAISE DEBUG 'INSERT time (s): %', clock_timestamp() - t;
+
+  -- Deal with modified rows: ids in source and dest but different hashes
+  t := clock_timestamp();
+  update_set_clause :=  @extschema@.__CDB_GetUpdateSetClause(colnames, '_changed');
+  dst_colnames := array_to_string(@extschema@.__CDB_QualifyColumns('_dst', colnames), ',');
+  src_colnames := array_to_string(@extschema@.__CDB_QualifyColumns('_src', colnames), ',');
+  EXECUTE format('
+      UPDATE %1$s _update SET %2$s
+      FROM (
+        SELECT _src.* FROM %3$s _src JOIN %1$s _dst ON (_dst.cartodb_id = _src.cartodb_id)
+        WHERE  md5(ROW(%4$s)::text) <> md5(ROW(%5$s)::text)
+      ) _changed
+      WHERE _update.cartodb_id = _changed.cartodb_id;
+  ', fq_dest_table, update_set_clause, src_table, dst_colnames, src_colnames);
+  GET DIAGNOSTICS num_rows = ROW_COUNT;
+  RAISE NOTICE 'MODIFIED % row(s)', num_rows;
+  RAISE DEBUG 'UPDATE time (s): %', clock_timestamp() - t;
+END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_TableMetadata.sql

@@ -43,7 +43,7 @@ BEGIN
   );
 
   WITH nv as (
-    SELECT TG_RELID as tabname, NOW() as t
+    SELECT TG_RELID as tabname, now() as t
   ), updated as (
     UPDATE @extschema@.CDB_TableMetadata x SET updated_at = nv.t
     FROM nv WHERE x.tabname = nv.tabname
@@ -55,8 +55,11 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 --
 -- Trigger invalidating varnish whenever CDB_TableMetadata
@@ -116,15 +119,18 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DROP TRIGGER IF EXISTS table_modified ON @extschema@.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table
 --       oid (regclass) to its name
 CREATE TRIGGER table_modified AFTER INSERT OR UPDATE
 ON @extschema@.CDB_TableMetadata FOR EACH ROW EXECUTE PROCEDURE
- _CDB_TableMetadata_Updated();
+    @extschema@._CDB_TableMetadata_Updated();
 
 
 -- similar to TOUCH(1) in unix filesystems but for table in cdb_tablemetadata

scripts-available/CDB_Username.sql

@@ -2,5 +2,9 @@
 CREATE OR REPLACE FUNCTION @extschema@.CDB_Username()
 RETURNS text
 AS $$
-  SELECT @extschema@.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
-$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;
+  SELECT @extschema@.CDB_Conf_GetConf(concat('api_keys_', session_user))->>'username';
+$$  LANGUAGE SQL
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;

scripts-enabled/300-CDB_OAuth.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_OAuth.sql

scripts-enabled/CDB_SyncTable.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_SyncTable.sql

test/CDB_AnalysisCheckTest.sql

@@ -7,7 +7,7 @@ SELECT _CDB_AnalysisDataSize('public');
 CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int);
 CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int);
 CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int);
-SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisTablesInSchema('public') t ORDER BY t;
 SELECT _CDB_AnalysisDataSize('public');
 SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
 SELECT CDB_SetUserQuotaInBytes(1);

test/CDB_OAuth.sql

@@ -0,0 +1,177 @@
+-- Create user and enable OAuth event trigger
+\set QUIET on
+SET client_min_messages TO error;
+DROP ROLE IF EXISTS "creator_role";
+CREATE ROLE "creator_role" LOGIN;
+DROP ROLE IF EXISTS "ownership_role";
+CREATE ROLE "ownership_role" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "creator_role";
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[]}');
+SET SESSION AUTHORIZATION "creator_role";
+SET client_min_messages TO notice;
+\set QUIET off
+
+-- First part without event trigger
+
+CREATE TABLE test(id INT);
+INSERT INTO test VALUES(1);
+CREATE TABLE test_tablesas AS SELECT * FROM test;
+CREATE VIEW test_view AS SELECT * FROM test;
+CREATE MATERIALIZED VIEW test_mview AS SELECT * FROM test;
+SELECT * INTO test_selectinto FROM test;
+
+SELECT * FROM test;
+SELECT * FROM test_tablesas;
+SELECT * FROM test_view;
+SELECT * FROM test_mview;
+SELECT * FROM test_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT * FROM test;
+SELECT * FROM test_tablesas;
+SELECT * FROM test_view;
+SELECT * FROM test_mview;
+SELECT * FROM test_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test_tablesas;
+DROP VIEW test_view;
+DROP MATERIALIZED VIEW test_mview;
+DROP TABLE test_selectinto;
+DROP TABLE test;
+
+-- Second part with event trigger but without ownership_role_name in cdb_conf
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_EnableOAuthReassignTablesTrigger();
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test2(id INT);
+INSERT INTO test2 VALUES(1);
+CREATE TABLE test2_tablesas AS SELECT * FROM test2;
+CREATE VIEW test2_view AS SELECT * FROM test2;
+CREATE MATERIALIZED VIEW test2_mview AS SELECT * FROM test2;
+SELECT * INTO test2_selectinto FROM test2;
+
+SELECT * FROM test2;
+SELECT * FROM test2_tablesas;
+SELECT * FROM test2_view;
+SELECT * FROM test2_mview;
+SELECT * FROM test2_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT * FROM test2;
+SELECT * FROM test2_tablesas;
+SELECT * FROM test2_view;
+SELECT * FROM test2_mview;
+SELECT * FROM test2_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test2_tablesas;
+DROP VIEW test2_view;
+DROP MATERIALIZED VIEW test2_mview;
+DROP TABLE test2_selectinto;
+DROP TABLE test2;
+
+-- Third part with event trigger but with empty ownership_role_name in cdb_conf
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": ""}');
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test3(id INT);
+INSERT INTO test3 VALUES(1);
+CREATE TABLE test3_tablesas AS SELECT * FROM test3;
+CREATE VIEW test3_view AS SELECT * FROM test3;
+CREATE MATERIALIZED VIEW test3_mview AS SELECT * FROM test3;
+SELECT * INTO test3_selectinto FROM test3;
+
+SELECT * FROM test3;
+SELECT * FROM test3_tablesas;
+SELECT * FROM test3_view;
+SELECT * FROM test3_mview;
+SELECT * FROM test3_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT * FROM test3;
+SELECT * FROM test3_tablesas;
+SELECT * FROM test3_view;
+SELECT * FROM test3_mview;
+SELECT * FROM test3_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test3_tablesas;
+DROP VIEW test3_view;
+DROP MATERIALIZED VIEW test3_mview;
+DROP TABLE test3_selectinto;
+DROP TABLE test3;
+
+-- Fourth part with the event trigger active and configured
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": "ownership_role"}');
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test4(id INT);
+INSERT INTO test4 VALUES(1);
+CREATE TABLE test4_tablesas AS SELECT * FROM test4;
+CREATE VIEW test4_view AS SELECT * FROM test4;
+CREATE MATERIALIZED VIEW test4_mview AS SELECT * FROM test4;
+SELECT * INTO test4_selectinto FROM test4;
+
+SELECT * FROM test4;
+SELECT * FROM test4_tablesas;
+SELECT * FROM test4_view;
+SELECT * FROM test4_mview;
+SELECT * FROM test4_selectinto;
+
+\set QUIET on
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT * FROM test4;
+SELECT * FROM test4_tablesas;
+SELECT * FROM test4_view;
+SELECT * FROM test4_mview;
+SELECT * FROM test4_selectinto;
+
+-- Ownership role drops the tables
+DROP TABLE test4_tablesas;
+DROP VIEW test4_view;
+DROP MATERIALIZED VIEW test4_mview;
+DROP TABLE test4_selectinto;
+DROP TABLE test4;
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableOAuthReassignTablesTrigger();
+DROP ROLE "ownership_role";
+REVOKE ALL ON SCHEMA cartodb FROM "creator_role";
+DROP ROLE "creator_role";
+DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role';
+\set QUIET off

test/CDB_OAuth_expect

@@ -0,0 +1,90 @@
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for relation test
+ERROR:  permission denied for relation test_tablesas
+ERROR:  permission denied for relation test_view
+ERROR:  permission denied for relation test_mview
+ERROR:  permission denied for relation test_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for relation test2
+ERROR:  permission denied for relation test2_tablesas
+ERROR:  permission denied for relation test2_view
+ERROR:  permission denied for relation test2_mview
+ERROR:  permission denied for relation test2_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for relation test3
+ERROR:  permission denied for relation test3_tablesas
+ERROR:  permission denied for relation test3_view
+ERROR:  permission denied for relation test3_mview
+ERROR:  permission denied for relation test3_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+

test/CDB_OAuth_expect.pg11

@@ -0,0 +1,90 @@
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for table test
+ERROR:  permission denied for table test_tablesas
+ERROR:  permission denied for view test_view
+ERROR:  permission denied for materialized view test_mview
+ERROR:  permission denied for table test_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for table test2
+ERROR:  permission denied for table test2_tablesas
+ERROR:  permission denied for view test2_view
+ERROR:  permission denied for materialized view test2_mview
+ERROR:  permission denied for table test2_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+ERROR:  permission denied for table test3
+ERROR:  permission denied for table test3_tablesas
+ERROR:  permission denied for view test3_view
+ERROR:  permission denied for materialized view test3_mview
+ERROR:  permission denied for table test3_selectinto
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+

test/CDB_SyncTableTest.sql

@@ -0,0 +1,94 @@
+-- Setup: create and populate a table to test the syncs
+\set QUIET on
+BEGIN;
+SET client_min_messages TO error;
+CREATE TABLE test_sync_source (
+  cartodb_id bigint,
+  lat double precision,
+  lon double precision,
+  name text
+);
+INSERT INTO test_sync_source VALUES
+  (1, 1.0, 1.0, 'foo'),
+  (2, 2.0, 2.0, 'bar'),
+  (3, 3.0, 3.0, 'patata'),
+  (4, 4.0, 4.0, 'melon');
+SET client_min_messages TO notice;
+\set QUIET off
+
+
+\echo 'First table sync: it should be simply just copied to the destination'
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Next table sync: there shall be no changes'
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Remove a row from the source and check it is deleted from the dest table'
+DELETE FROM test_sync_source WHERE cartodb_id = 3;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Insert a new row and check that it is inserted in the dest table'
+INSERT INTO test_sync_source VALUES (5, 5.0, 5.0, 'sandia');
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Modify row and check that it is modified in the dest table'
+UPDATE test_sync_source SET name = 'cantaloupe' WHERE cartodb_id = 4;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest');
+
+\echo 'Sanity check: the end result is the same source table'
+SELECT * FROM test_sync_source ORDER BY cartodb_id;
+SELECT * FROM test_sync_dest ORDER BY cartodb_id;
+
+
+\echo 'It shall exclude geom columns if instructed to do so'
+\set QUIET on
+SET client_min_messages TO error;
+SELECT cartodb.CDB_SetUserQuotaInBytes(0); -- Set user quota to infinite
+SELECT cartodb.CDB_CartodbfyTable('test_sync_source');
+SELECT cartodb.CDB_CartodbfyTable('test_sync_dest');
+UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecoding"
+\set QUIET off
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}');
+SELECT * FROM test_sync_source ORDER BY cartodb_id;
+SELECT * FROM test_sync_dest ORDER BY cartodb_id;
+
+\echo 'It will work with schemas that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+CREATE SCHEMA "sch-ema";
+CREATE TABLE "test_sync_source2" AS SELECT * FROM test_sync_source;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test_sync_source2', 'sch-ema', 'test_sync_dest');
+INSERT INTO test_sync_source2(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM test_sync_source2 WHERE cartodb_id = 4;
+UPDATE test_sync_source2 SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source2', 'sch-ema', 'test_sync_dest');
+
+\echo 'It will work with table names that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+CREATE TABLE "test-sync-source" AS SELECT * FROM test_sync_source;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test-sync-source', 'public', 'test-sync-dest');
+INSERT INTO "test-sync-source"(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM "test-sync-source" WHERE cartodb_id = 4;
+UPDATE "test-sync-source" SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test-sync-source', 'public', 'test-sync-dest');
+
+\echo 'It will work with column names that need quoting'
+\set QUIET on
+SET client_min_messages TO error;
+ALTER TABLE test_sync_source ADD COLUMN "a-column" int;
+\set QUIET off
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest2');
+INSERT INTO test_sync_source(cartodb_id, lat, lon, name) VALUES (6, 6.0, 6.0, 'papaya');
+DELETE FROM test_sync_source WHERE cartodb_id = 4;
+UPDATE test_sync_source SET lat = 2.5 WHERE cartodb_id = 2;
+SET client_min_messages TO notice;
+SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest2');
+
+-- Cleanup
+ROLLBACK;

test/CDB_SyncTableTest_expect

@@ -0,0 +1,94 @@
+First table sync: it should be simply just copied to the destination
+NOTICE:  INSERTED 4 row(s)
+
+Next table sync: there shall be no changes
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Remove a row from the source and check it is deleted from the dest table
+DELETE 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Insert a new row and check that it is inserted in the dest table
+INSERT 0 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 0 row(s)
+
+Modify row and check that it is modified in the dest table
+UPDATE 1
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 0 row(s)
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+Sanity check: the end result is the same source table
+1|1|1|foo
+2|2|2|bar
+4|4|4|cantaloupe
+5|5|5|sandia
+1|1|1|foo
+2|2|2|bar
+4|4|4|cantaloupe
+5|5|5|sandia
+It shall exclude geom columns if instructed to do so
+0
+test_sync_source
+test_sync_dest
+SET
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  DELETED 0 row(s)
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  INSERTED 0 row(s)
+NOTICE:  cdb_invalidate_varnish(public.test_sync_dest) called
+NOTICE:  MODIFIED 0 row(s)
+
+1|||1|1|foo
+2|||2|2|bar
+4|||4|4|cantaloupe
+5|||5|5|sandia
+1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo
+2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar
+4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe
+5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia
+It will work with schemas that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test_sync_dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+It will work with table names that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test-sync-dest" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+It will work with column names that need quoting
+
+INSERT 0 1
+DELETE 1
+UPDATE 1
+SET
+NOTICE:  relation "test_sync_dest2" already exists, skipping
+NOTICE:  DELETED 1 row(s)
+NOTICE:  INSERTED 1 row(s)
+NOTICE:  MODIFIED 1 row(s)
+
+ROLLBACK

test/README

@@ -6,6 +6,14 @@ Example, to add a test for CDB_Something function, you'd add:
  - CDB_SomethingTest.sql
  - CDB_SomethingTest_expect
 
+In case you need multiple expects of a test for different versions you have
+to add .pg$(VERSION) at the end of the file.
+
+For example if you want an expect file for PG11 you need to have two expect files:
+
+  - CDB_SomethingTest_expect
+  - CDB_SomethingTest_expect.pg11
+
 To easy the generation of the expected file you can initially omit it,
 then run "make -C .. installcheck" from the top-level dir and copy
 ../results/test/CDB_SomethingTest.out to CDB_SomethingTest_expect chopping

test/extension/test.sh

@@ -362,7 +362,7 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
 
 function test_cdb_tablemetadatatouch_fully_qualifies_names() {
     sql postgres "CREATE TABLE touch_invalidations (table_name text);"
-    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into public.touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
 
     #default schema
     sql "CREATE TABLE touch_example (a int);"
@@ -586,6 +586,73 @@ test_extension|public|"local-table-with-dashes"'
     sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) < now()" should 't'
     sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) > (now() - interval '1 minute')" should 't'
 
+    # Check CDB_Get_Foreign_Updated_At is robust to unimported CDB_TableMetadata
+    sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;"
+    sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't'
+
+
+    # Check user-defined FDW's
+    # Set up a user foreign server
+    read -d '' ufdw_config <<- EOF
+{
+   "server": {
+     "extensions": "postgis",
+     "dbname": "fdw_target",
+     "host": "localhost",
+     "port": ${PGPORT:-5432}
+   },
+   "user_mapping": {
+     "user": "fdw_user",
+     "password": "foobarino"
+   }
+}
+EOF
+    sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');"
+
+    # Grant a user access to that FDW, and to grant to others
+    sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;'
+
+    # Set up a user foreign table
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');"
+
+    # Check that the table can be accessed by the owner/creator
+    sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;'
+    sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+
+    # Check that a role with no permissions cannot use the FDW to access a remote table
+    sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails
+
+    # Check that the table can be accessed by some other user by granting the role
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;'
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;'
+
+    # Check that the table can be accessed by org members
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');"
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');"
+
+    # By default publicuser cannot access the FDW
+    sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted
+    sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;'
+
+    # If there are dependent objects, we cannot drop the foreign server
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails
+    sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;'
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')"
+
+    # But if there are, we can set the force flag to true to drop everything (defaults to false)
+    sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');"
+    sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;'
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');"
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)"
+
+
+    # Teardown
     DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo2 FROM fdw_user;'