Increase version number to 0.32.0

Fix OAuth ownership re-assignation for functions

.travis.yml

@@ -8,26 +8,28 @@ env:
     - PGDATABASE=postgres
     - PGOPTIONS='-c client_min_messages=NOTICE'
     - PGPORT=5432
-    - POSTGIS_VERSION="2.5"
 
-  matrix:
-    - POSTGRESQL_VERSION="9.6"
-    - POSTGRESQL_VERSION="10"
-    - POSTGRESQL_VERSION="11"
+jobs:
+  include:
+    - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
+  
 
-
-before_install:
+script:
   - sudo service postgresql stop;
   - sudo apt-get remove postgresql* -y
   - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
   - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
-  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
+  # For pre12, install plpython2. For PG12 install plpython3
+  - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi;
   - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
   - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
-  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password
   - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
-  - sudo pip install redis==2.4.9
-script:
   - make
   - sudo make install
   - make installcheck

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.28.1
+EXTVERSION = 0.32.0
 
 SED = sed
 AWK = awk
@@ -101,6 +101,10 @@ UPGRADABLE = \
   0.27.2 \
   0.28.0 \
   0.28.1 \
+  0.29.0 \
+  0.30.0 \
+  0.31.0 \
+  0.32.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -124,17 +128,25 @@ EXTRA_CLEAN = cartodb_version.sql
 DOCS = README.md
 REGRESS_OLD = $(wildcard test/*.sql)
 REGRESS_LEGACY = $(REGRESS_OLD:.sql=)
-REGRESS = test_setup $(REGRESS_LEGACY)
+REGRESS = test/test_setup $(REGRESS_LEGACY)
 
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
+
 $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
 	$(SED) 	-e 's/@extschema@/cartodb/g' \
-		-e "s/@postgisschema@/public/g" >> $@
+		-e 's/@postgisschema@/public/g' \
+		-e 's/plpythonu/$(PLPYTHONU)/g' >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 
@@ -148,10 +160,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 	cp $< $@
 
 $(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
 
 # Needed for consistent `echo` results with backslashes
 SHELL = bash
@@ -160,6 +172,10 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
 	mkdir -p expected/test/
 	mkdir -p results/test/
+	cat sql/test_setup.sql | \
+			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' \
+			> sql/test/test_setup.sql
+	cp sql/test_setup_expect expected/test/test_setup.out
 	for f in $(REGRESS_OLD); do \
 		tn=`basename $${f} .sql`; \
 		of=sql/test/$${tn}.sql; \
@@ -168,7 +184,7 @@ legacy_regress: $(REGRESS_OLD) Makefile
 		echo '\t' >> $${of}; \
 		echo '\set QUIET off' >> $${of}; \
 		cat $${f} | \
-			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \
+			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' >> $${of}; \
 		exp=expected/test/$${tn}.out; \
 		echo '\set ECHO none' > $${exp}; \
 		cat test/$${tn}_expect >> $${exp}; \
@@ -180,7 +196,11 @@ test_organization:
 test_extension_new:
 	bash test/extension/test.sh
 
-legacy_tests: legacy_regress
+legacy_tests: legacy_regress $(EXTENSION)--unpackaged--$(EXTVERSION).sql
+
+PGREGRESS := $(shell dirname `$(PG_CONFIG) --pgxs`)/../../src/test/regress/pg_regress
+regress: legacy_tests
+	$(PGREGRESS) --inputdir=./ --bindir='/usr/bin'    --dbname=contrib_regression $(REGRESS)
 
 installcheck: legacy_tests test_extension_new test_organization
 

NEWS.md

@@ -1,3 +1,23 @@
+0.32.0 (2019-11-08)
+* Fix oAuth ownership re-assignation for functions
+* Some fixes for PG12.
+* Make PG12 depend on plpython3u instead of plpythonu
+* CDB_UserDataSize is now compatible with postgis 3 without postgis_raster.
+* Makefile: Add regress target (checks regress tests without needing to install the extension)
+
+0.31.0 (2019-10-08)
+* Ghost tables: Add missing tags (#370)
+* Set search_path in security definer functions.
+
+0.30.0 (2019-07-17)
+* Added new admin functions to connect CARTO with user FDW's (#369)
+
+0.29.0 (2019-07-15)
+* Added new function CDB_OAuth:
+  * Install event trigger to check for table/view/sequence/function creation
+  * Reassign the ownership of new objects to a defined role in the cdb_conf
+* Changed MakeFile to support different expects for differents PG versions
+
 0.28.1 (2019-07-04)
 * Avoid temporary tables creation in CDB_SyncTable (#366)
 * Make CDB_Get_Foreign_Updated_At robust to missing CDB_TableMetadata (#362)

README.md

@@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/
 Dependencies
 ------------
 
- * PostgreSQL 9.6+ (with plpythonu extension and xml support)
+ * PostgreSQL 9.6+ (with plpythonu extension and xml support). For PostgreSQL 12+ plpython3u is required instead of plpythonu.
  * [PostGIS extension](http://postgis.net)
  * Python with [Redis module](https://pypi.org/project/redis/)
 

expected/test_setup.out

@@ -1,6 +1,5 @@
-CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
+SET client_min_messages TO error;
+CREATE EXTENSION cartodb CASCADE;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN

scripts-available/CDB_AnalysisCheck.sql

@@ -6,8 +6,11 @@ $$
 BEGIN
   RETURN @extschema@.CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
 END;
-$$
-LANGUAGE 'plpgsql' STABLE PARALLEL SAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 
 -- Get the factor (fraction of the quota) for Camshaft cached analysis tables

scripts-available/CDB_EstimateRowCount.sql

@@ -12,7 +12,12 @@ BEGIN
     EXECUTE Format('ANALYZE %s;', reloid);
   END IF;
 END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    VOLATILE
+    STRICT
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Return a row count estimate of the result of a query using statistics
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EstimateRowCount(query text)

scripts-available/CDB_ForeignTable.sql

@@ -139,6 +139,176 @@ $$
 LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
 
 
+-- Produce a valid DB name for objects created for the user FDW's
+CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME)
+RETURNS NAME AS $$
+  -- Note on input we use %s and on output we use %I, in order to
+  -- avoid double escaping
+  SELECT format('cdb_fdw_%s', fdw_input_name)::NAME;
+$$
+LANGUAGE sql IMMUTABLE PARALLEL SAFE;
+
+-- A function to set up a user-defined foreign data server
+-- It does not read from CDB_Conf.
+-- Only superuser roles can invoke it successfully
+--
+-- Sample call:
+-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{
+--    "server": {
+--      "extensions": "postgis",
+--      "dbname": "testdb",
+--      "host": "myhostname.us-east-2.rds.amazonaws.com",
+--      "port": "5432"
+--    },
+--    "user_mapping": {
+--      "user": "fdw_user",
+--      "password": "secret"
+--    }
+-- }');
+--
+-- Underneath it will:
+--   * Set up postgresql_fdw
+--   * Create a server with the name 'cdb_fdw_amazon'
+--   * Create a role called 'cdb_fdw_amazon' to manage access
+--   * Create a user mapping with that role 'cdb_fdw_amazon'
+--   * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign
+--     tables over there
+--
+-- It is the responsibility of the superuser to grant that role to either:
+--   * Nobody
+--   * Specific roles: GRANT amazon TO role_name;
+--   * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon');
+--   * The publicuser: GRANT cdb_fdw_amazon TO publicuser;
+CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json)
+RETURNS void AS $$
+DECLARE
+  row record;
+  option record;
+  fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+BEGIN
+  -- TODO: refactor with original function
+  -- This function tries to be as idempotent as possible, by not creating anything more than once
+  -- (not even using IF NOT EXIST to avoid throwing warnings)
+  IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
+    CREATE EXTENSION postgres_fdw;
+    RAISE NOTICE 'Created postgres_fdw EXTENSION';
+  END IF;
+  -- Create FDW first if it does not exist
+  IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name)
+    THEN
+    EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name);
+    RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name;
+  END IF;
+
+  -- Set FDW settings
+  FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
+    LOOP
+      IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key)
+        THEN
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value);
+      ELSE
+        EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value);
+      END IF;
+    END LOOP;
+
+    -- Create specific role for this
+    IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN
+       EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name);
+       RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name;
+    END IF;
+
+    -- Transfer ownership of the server to the fdw role
+    EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
+
+    -- Create user mapping
+    -- NOTE: we use a PUBLIC user mapping but control access to the SERVER
+    -- so that we don't need to create a mapping for every user nor store credentials elsewhere
+    IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN
+        EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name);
+        RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name;
+    END IF;
+
+    -- Update user mapping settings
+    FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP
+        IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value);
+        ELSE
+          EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value);
+        END IF;
+    END LOOP;
+
+    -- Grant usage on the wrapper and server to the fdw role
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name);
+    RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name;
+    EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name);
+    RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name;
+
+    -- Create schema if it does not exist.
+    IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN
+      EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name);
+      RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name;
+    END IF;
+
+    -- Give the fdw role ownership over the schema
+    EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
+    RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name;
+
+    -- TODO: Bring here the remote cdb_tablemetadata
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
+-- A function to drop a user-defined foreign server and all related objects
+-- It does not read from CDB_Conf
+-- It must be executed with a superuser role to succeed
+--
+-- Sample call:
+--   SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon')
+--
+-- Note: if there's any dependent object (i.e. foreign table) this call will fail
+CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false)
+RETURNS void AS $$
+DECLARE
+    fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+    cascade_clause NAME;
+BEGIN
+    CASE force
+        WHEN true THEN
+            cascade_clause := 'CASCADE';
+        ELSE
+            cascade_clause := 'RESTRICT';
+    END CASE;
+
+    EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Dropped schema %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name);
+    RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Dropped foreign server %', fdw_objects_name;
+    EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause);
+    RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name;
+    EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name);
+    RAISE NOTICE 'Dropped role %', fdw_objects_name;
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
+-- Set up a user foreign table
+-- E.g:
+--   SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable');
+--   SELECT * FROM amazon.my_table;
+CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME)
+RETURNS void AS $$
+DECLARE
+    fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
+BEGIN
+  EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name);
+  --- Grant SELECT to fdw role
+  EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name);
+END
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+
 CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid)
 RETURNS TEXT AS $$
     SELECT option_value FROM pg_options_to_table((

scripts-available/CDB_GhostTables.sql

@@ -63,7 +63,11 @@ AS $$
     PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
     RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Trigger function to call CDB_LinkGhostTables()
 CREATE OR REPLACE FUNCTION @extschema@._CDB_LinkGhostTablesTrigger()
@@ -76,7 +80,11 @@ AS $$
     PERFORM @extschema@.CDB_LinkGhostTables(ddl_tag);
     RETURN NULL;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Event trigger to save the current transaction in @extschema@.cdb_ddl_execution
 CREATE OR REPLACE FUNCTION @extschema@.CDB_SaveDDLTransaction()
@@ -85,7 +93,11 @@ AS $$
   BEGIN
     INSERT INTO @extschema@.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT ON CONSTRAINT cdb_ddl_execution_pkey DO NOTHING;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Creates the trigger on DDL events to link ghost tables
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableGhostTablesTrigger()
@@ -106,7 +118,33 @@ AS $$
 
     CREATE EVENT TRIGGER link_ghost_tables
     ON ddl_command_end
-    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW', 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE', 'DROP FOREIGN TABLE')
+    WHEN TAG IN ('CREATE TABLE',
+                'SELECT INTO',
+                'DROP TABLE',
+                'ALTER TABLE',
+
+                'CREATE TRIGGER',
+                'DROP TRIGGER',
+                'ALTER TRIGGER',
+
+                'CREATE VIEW',
+                'DROP VIEW',
+                'ALTER VIEW',
+
+                'CREATE FOREIGN TABLE',
+                'ALTER FOREIGN TABLE',
+                'DROP FOREIGN TABLE',
+
+                'ALTER MATERIALIZED VIEW',
+                'CREATE MATERIALIZED VIEW',
+                'DROP MATERIALIZED VIEW',
+
+                'IMPORT FOREIGN SCHEMA',
+
+                'DROP EXTENSION',
+                'DROP SCHEMA',
+                'DROP SERVER',
+                'DROP TYPE')
     EXECUTE PROCEDURE @extschema@.CDB_SaveDDLTransaction();
   END;
 $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Groups_API.sql

@@ -22,59 +22,91 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text)
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(group_name))
 
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(old_group_name))
     body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -89,13 +121,20 @@ FUNCTION @extschema@._CDB_Group_Table_GrantPermission_API(group_name text, usern
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
     body = '{ "access": "%s" }' % access
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -110,12 +149,19 @@ FUNCTION @extschema@._CDB_Group_Table_RevokeAllPermission_API(group_name text, u
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -161,7 +207,10 @@ CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
     RETURNS int AS
 $$
-    import httplib
+    try:
+        import httplib as client
+    except:
+        from http import client
 
     params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0]
     if params['host'] is None:
@@ -174,17 +223,17 @@ $$
     last_err = None
     while retry > 0:
       try:
-        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout'])
         database_name = plpy.execute("select current_database();")[0]['current_database']
-        client.request(method, url.format(database_name), body, headers)
-        response = client.getresponse()
+        conn.request(method, url.format(database_name), body, headers)
+        response = conn.getresponse()
         assert response.status in valid_return_codes
         return response.status
       except Exception as err:
         retry -= 1
         last_err = err
         plpy.warning('Retrying after: ' + str(err))
-        client = SD['groups_api_client'] = None
+        conn = SD['groups_api_client'] = None
 
     if last_err is not None:
       plpy.error('Fatal Group API error: ' + str(last_err))

scripts-available/CDB_OAuth.sql

@@ -0,0 +1,65 @@
+-- Function that reassign the owner of a table to their ownership_role
+CREATE OR REPLACE FUNCTION @extschema@.CDB_OAuthReassignTableOwnerOnCreation()
+  RETURNS event_trigger
+  AS $$
+DECLARE
+    obj record;
+    owner_role text;
+    creator_role text;
+BEGIN
+    FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands()
+    LOOP
+      RAISE DEBUG '% ddl object: % % % %',
+                  tg_tag,
+                  obj.command_tag,
+                  obj.object_type,
+                  obj.schema_name,
+                  obj.object_identity;
+      IF obj.object_type = 'function' THEN
+        SELECT rolname FROM pg_proc JOIN pg_roles ON proowner = pg_roles.oid WHERE pg_proc.oid = obj.objid INTO creator_role;
+      ELSE
+	      SELECT rolname FROM pg_class JOIN pg_roles ON relowner = pg_roles.oid WHERE pg_class.oid = obj.objid INTO creator_role;
+      END IF;
+      SELECT value->>'ownership_role_name' from @extschema@.CDB_Conf_GetConf('api_keys_' || quote_ident(creator_role)) value INTO owner_role;
+      IF owner_role IS NULL OR owner_role = '' THEN
+        RAISE DEBUG 'owner_role not found';
+        CONTINUE;
+      ELSE
+        EXECUTE 'ALTER ' || obj.object_type || ' ' || obj.object_identity || ' OWNER TO ' || quote_ident(owner_role);
+        IF obj.object_type = 'function' THEN
+          EXECUTE 'GRANT ALL ON FUNCTION ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        ELSE
+          EXECUTE 'GRANT ALL ON ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        END IF;        
+        RAISE DEBUG 'Changing ownership from % to %', creator_role, owner_role;
+      END IF;
+    END LOOP;
+END;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
+
+-- Creates the trigger on DDL events in order to reassign the owner
+CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableOAuthReassignTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
+
+    CREATE EVENT TRIGGER oauth_reassign_tables_trigger
+    ON ddl_command_end
+    WHEN TAG IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'CREATE VIEW', 'CREATE FOREIGN TABLE', 'CREATE MATERIALIZED VIEW', 'CREATE SEQUENCE', 'CREATE FUNCTION')
+    EXECUTE PROCEDURE @extschema@.CDB_OAuthReassignTableOwnerOnCreation();
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+-- Deletes the trigger on DDL events in order to reassign the owner
+CREATE OR REPLACE FUNCTION @extschema@.CDB_DisableOAuthReassignTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS oauth_reassign_tables_trigger;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Organizations.sql

@@ -169,3 +169,30 @@ BEGIN
     EXECUTE 'SELECT @extschema@.CDB_Organization_Remove_Access_Permission(''' || from_schema || ''', ''' || table_name || ''', ''' || @extschema@.CDB_Organization_Member_Group_Role_Member_Name() || ''');';
 END
 $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
+
+
+--------------------------------------------------------------------------------
+-- Role management
+--------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name)
+RETURNS VOID AS $$
+DECLARE
+    org_role TEXT;
+BEGIN
+    org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
+    EXECUTE format('GRANT %I TO %I', role_name, org_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
+
+
+CREATE OR REPLACE
+FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name)
+RETURNS VOID AS $$
+DECLARE
+    org_role TEXT;
+BEGIN
+    org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
+    EXECUTE format('REVOKE %I FROM %I', role_name, org_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Overviews.sql

@@ -318,7 +318,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 -- This function is declared SECURITY DEFINER so it executes with the privileges
 -- of the function creator to have a chance to alter the privileges of the
 -- overview table to match those of the dataset. It will only perform any change
--- if the overview table belgons to the same scheme as the dataset and it
+-- if the overview table belongs to the same scheme as the dataset and it
 -- matches the scheme naming for overview tables.
 CREATE OR REPLACE FUNCTION @extschema@._CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
 RETURNS VOID
@@ -362,7 +362,11 @@ AS $$
       -- it should be done here (CDB_Overviews would consume such metadata)
     END IF;
   END
-$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE PLPGSQL
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Dataset attributes (column names other than the
 -- CartoDB primary key and geometry columns) which should be aggregated
@@ -393,7 +397,7 @@ DECLARE
   attr_list TEXT;
 BEGIN
   SELECT string_agg(s.c, ',') FROM (
-    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
+    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -550,7 +554,7 @@ DECLARE
 BEGIN
   SELECT string_agg(@extschema@._CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
   FROM (
-    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
+    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -658,7 +662,7 @@ AS $$
       offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
     END IF;
 
-    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+    point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
 
     -- compute the resulting columns in the same order as in the base table
     WITH cols AS (
@@ -669,7 +673,7 @@ AS $$
           Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom)
         WHEN 'the_geom_webmercator' THEN
            Format('%s AS the_geom_webmercator', point_geom)
-        ELSE c
+        ELSE c::text
         END AS column
         FROM @extschema@.CDB_ColumnNames(reloid) c
     )
@@ -796,7 +800,7 @@ AS $$
           '@postgisschema@.ST_Transform(@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom'
         WHEN 'the_geom_webmercator' THEN
           '@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator'
-        ELSE c
+        ELSE c::text
         END AS column
         FROM CDB_ColumnNames(reloid) c
     )
@@ -920,7 +924,7 @@ AS $$
       SELECT
         CASE c
         WHEN 'cartodb_id' THEN 'cartodb_id'
-        ELSE c
+        ELSE c::text
         END AS column
         FROM @extschema@.CDB_ColumnNames(reloid) c
     )

scripts-available/CDB_Quota.sql

@@ -20,33 +20,50 @@ RETURNS bigint AS
 $$
 DECLARE
   total_size INT8;
+  raster_available BOOLEAN;
+  raster_read_query TEXT;
 BEGIN
+  -- Postgis 3+ might not install raster
+  raster_available := EXISTS (
+        SELECT 1
+        FROM   pg_views
+        WHERE  schemaname = '@postgisschema@'
+        AND    viewname = 'raster_overviews'
+   );
+
+   IF raster_available THEN
+       raster_read_query := Format('SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews
+                        WHERE o_table_schema = ''%I'' AND o_table_catalog = current_database()', schema_name);
+   ELSE
+       raster_read_query := 'SELECT NULL::text AS o_table_name, NULL::text AS r_table_name';
+   END IF;
+   EXECUTE Format('
   WITH raster_tables AS (
-    SELECT o_table_name, r_table_name FROM raster_overviews
-      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
+    %s
   ),
   user_tables AS (
-    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(schema_name)
+    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(''%I'')
   ),
   table_cat AS (
     SELECT
       table_name,
       (
         EXISTS(select * from raster_tables where o_table_name = table_name)
-        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || '[\w\d]*'
+        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || ''[\w\d]*''
       ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(schema_name, table_name)))) table_size,
+    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(''%I'', table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1
 	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
       END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
   )
-  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
+  SELECT sum(table_size*multiplier)::int8 FROM sizes
+  ', raster_read_query, schema_name, schema_name) INTO total_size;
 
   IF total_size IS NOT NULL THEN
     RETURN total_size;

scripts-available/CDB_TableMetadata.sql

@@ -43,7 +43,7 @@ BEGIN
   );
 
   WITH nv as (
-    SELECT TG_RELID as tabname, NOW() as t
+    SELECT TG_RELID as tabname, now() as t
   ), updated as (
     UPDATE @extschema@.CDB_TableMetadata x SET updated_at = nv.t
     FROM nv WHERE x.tabname = nv.tabname
@@ -55,8 +55,11 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 --
 -- Trigger invalidating varnish whenever CDB_TableMetadata
@@ -116,8 +119,11 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DROP TRIGGER IF EXISTS table_modified ON @extschema@.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table

scripts-available/CDB_Username.sql

@@ -2,5 +2,9 @@
 CREATE OR REPLACE FUNCTION @extschema@.CDB_Username()
 RETURNS text
 AS $$
-  SELECT @extschema@.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
-$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;
+  SELECT @extschema@.CDB_Conf_GetConf(concat('api_keys_', session_user))->>'username';
+$$  LANGUAGE SQL
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;

scripts-enabled/300-CDB_OAuth.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_OAuth.sql

sql/test_setup.sql

@@ -1,9 +1,14 @@
+\set ECHO none
+\set QUIET on
+SET client_min_messages TO error;
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
+CREATE SCHEMA cartodb;
+\i 'cartodb--unpackaged--@@VERSION@@.sql'
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
 $$ LANGUAGE 'plpgsql';
+\set QUIET off

sql/test_setup_expect

@@ -0,0 +1 @@
+\set ECHO none

test/CDB_GhostTables_expect

@@ -4,7 +4,7 @@ WARNING:  Invalidation service configuration not found. Skipping Ghost Tables li
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 BEGIN
@@ -12,7 +12,7 @@ cdb_ddl_execution
 0
 CREATE TABLE
 1
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
 COMMIT
 

test/CDB_OAuth.sql

@@ -0,0 +1,223 @@
+-- Create user and enable OAuth event trigger
+\set QUIET on
+SET client_min_messages TO error;
+
+-- The permission error changed between pre PG11 and post 11 (before everything was "relation", now it's "view", "table" and so on
+CREATE OR REPLACE FUNCTION catch_permission_error(query text)
+    RETURNS bool
+AS $$
+BEGIN
+    EXECUTE query;
+    RETURN FALSE;
+EXCEPTION
+    WHEN insufficient_privilege THEN
+        RETURN TRUE;
+    WHEN OTHERS THEN
+        RAISE WARNING 'Exception %', sqlstate;
+        RETURN FALSE;
+END
+$$ LANGUAGE 'plpgsql';
+
+DROP ROLE IF EXISTS "creator_role";
+CREATE ROLE "creator_role" LOGIN;
+DROP ROLE IF EXISTS "ownership_role";
+CREATE ROLE "ownership_role" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "creator_role";
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[]}');
+SET SESSION AUTHORIZATION "creator_role";
+SET client_min_messages TO notice;
+\set QUIET off
+
+-- First part without event trigger
+
+CREATE TABLE test(id INT);
+INSERT INTO test VALUES(1);
+CREATE TABLE test_tablesas AS SELECT * FROM test;
+CREATE VIEW test_view AS SELECT * FROM test;
+CREATE MATERIALIZED VIEW test_mview AS SELECT * FROM test;
+SELECT * INTO test_selectinto FROM test;
+CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+
+SELECT * FROM test;
+SELECT * FROM test_tablesas;
+SELECT * FROM test_view;
+SELECT * FROM test_mview;
+SELECT * FROM test_selectinto;
+SELECT test_function();
+-- Postgres grants default execute privilege on functions to PUBLIC. So in order to check the different permissions
+-- between creator and owner roles is not enough with performing a selection, we need to DROP the table (which only the owner can do)
+DROP FUNCTION test_function();
+
+\set QUIET on
+CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT 'denied_table', catch_permission_error($$SELECT * FROM test;$$);
+SELECT 'denied_tableas', catch_permission_error($$SELECT * FROM test_tablesas;$$);
+SELECT 'denied_view', catch_permission_error($$SELECT * FROM test_view;$$);
+SELECT 'denied_mview', catch_permission_error($$SELECT * FROM test_mview;$$);
+SELECT 'denied_selectinto', catch_permission_error($$SELECT * FROM test_selectinto;$$);
+SELECT test_function();
+SELECT 'denied_function', catch_permission_error($$DROP FUNCTION test_function();$$);
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test_tablesas;
+DROP VIEW test_view;
+DROP MATERIALIZED VIEW test_mview;
+DROP TABLE test_selectinto;
+DROP TABLE test;
+DROP FUNCTION test_function();
+
+-- Second part with event trigger but without ownership_role_name in cdb_conf
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_EnableOAuthReassignTablesTrigger();
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test2(id INT);
+INSERT INTO test2 VALUES(1);
+CREATE TABLE test2_tablesas AS SELECT * FROM test2;
+CREATE VIEW test2_view AS SELECT * FROM test2;
+CREATE MATERIALIZED VIEW test2_mview AS SELECT * FROM test2;
+SELECT * INTO test2_selectinto FROM test2;
+CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+
+SELECT * FROM test2;
+SELECT * FROM test2_tablesas;
+SELECT * FROM test2_view;
+SELECT * FROM test2_mview;
+SELECT * FROM test2_selectinto;
+SELECT test2_function();
+DROP FUNCTION test2_function();
+
+\set QUIET on
+CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT 'denied_table2', catch_permission_error($$SELECT * FROM test2;$$);
+SELECT 'denied_tableas2', catch_permission_error($$SELECT * FROM test2_tablesas;$$);
+SELECT 'denied_view2', catch_permission_error($$SELECT * FROM test2_view;$$);
+SELECT 'denied_mview2', catch_permission_error($$SELECT * FROM test2_mview;$$);
+SELECT 'denied_selectinto2', catch_permission_error($$SELECT * FROM test2_selectinto;$$);
+SELECT test2_function();
+SELECT 'denied_function2', catch_permission_error($$DROP FUNCTION test2_function();$$);
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test2_tablesas;
+DROP VIEW test2_view;
+DROP MATERIALIZED VIEW test2_mview;
+DROP TABLE test2_selectinto;
+DROP TABLE test2;
+DROP FUNCTION test2_function();
+
+-- Third part with event trigger but with empty ownership_role_name in cdb_conf
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": ""}');
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test3(id INT);
+INSERT INTO test3 VALUES(1);
+CREATE TABLE test3_tablesas AS SELECT * FROM test3;
+CREATE VIEW test3_view AS SELECT * FROM test3;
+CREATE MATERIALIZED VIEW test3_mview AS SELECT * FROM test3;
+SELECT * INTO test3_selectinto FROM test3;
+CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+
+SELECT * FROM test3;
+SELECT * FROM test3_tablesas;
+SELECT * FROM test3_view;
+SELECT * FROM test3_mview;
+SELECT * FROM test3_selectinto;
+SELECT test3_function();
+DROP FUNCTION test3_function();
+
+\set QUIET on
+CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT 'denied_table3', catch_permission_error($$SELECT * FROM test3;$$);
+SELECT 'denied_tableas3', catch_permission_error($$SELECT * FROM test3_tablesas;$$);
+SELECT 'denied_view3', catch_permission_error($$SELECT * FROM test3_view;$$);
+SELECT 'denied_mview3', catch_permission_error($$SELECT * FROM test3_mview;$$);
+SELECT 'denied_selectinto3', catch_permission_error($$SELECT * FROM test3_selectinto;$$);
+SELECT test3_function();
+SELECT 'denied_function3', catch_permission_error($$DROP FUNCTION test3_function();$$);
+
+\set QUIET on
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+DROP TABLE test3_tablesas;
+DROP VIEW test3_view;
+DROP MATERIALIZED VIEW test3_mview;
+DROP TABLE test3_selectinto;
+DROP TABLE test3;
+DROP FUNCTION test3_function();
+
+-- Fourth part with the event trigger active and configured
+
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_Conf_SetConf('api_keys_creator_role', '{"username": "creator_role", "permissions":[], "ownership_role_name": "ownership_role"}');
+SET SESSION AUTHORIZATION "creator_role";
+\set QUIET off
+
+CREATE TABLE test4(id INT);
+INSERT INTO test4 VALUES(1);
+CREATE TABLE test4_tablesas AS SELECT * FROM test4;
+CREATE VIEW test4_view AS SELECT * FROM test4;
+CREATE MATERIALIZED VIEW test4_mview AS SELECT * FROM test4;
+SELECT * INTO test4_selectinto FROM test4;
+CREATE FUNCTION test4_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
+
+SELECT * FROM test4;
+SELECT * FROM test4_tablesas;
+SELECT * FROM test4_view;
+SELECT * FROM test4_mview;
+SELECT * FROM test4_selectinto;
+SELECT test4_function();
+SELECT 'denied_function4', catch_permission_error($$DROP FUNCTION test4_function();$$);
+
+\set QUIET on
+SET SESSION AUTHORIZATION "ownership_role";
+\set QUIET off
+
+SELECT * FROM test4;
+SELECT * FROM test4_tablesas;
+SELECT * FROM test4_view;
+SELECT * FROM test4_mview;
+SELECT * FROM test4_selectinto;
+SELECT test4_function();
+
+-- Ownership role drops the tables
+DROP TABLE test4_tablesas;
+DROP VIEW test4_view;
+DROP MATERIALIZED VIEW test4_mview;
+DROP TABLE test4_selectinto;
+DROP TABLE test4;
+DROP FUNCTION test4_function();
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableOAuthReassignTablesTrigger();
+DROP ROLE "ownership_role";
+REVOKE ALL ON SCHEMA cartodb FROM "creator_role";
+DROP ROLE "creator_role";
+DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role';
+DROP FUNCTION catch_permission_error(text);
+\set QUIET off

test/CDB_OAuth_expect

@@ -0,0 +1,113 @@
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+CREATE FUNCTION
+1
+1
+1
+1
+1
+1
+DROP FUNCTION
+denied_table|t
+denied_tableas|t
+denied_view|t
+denied_mview|t
+denied_selectinto|t
+1
+denied_function|t
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+DROP FUNCTION
+NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+CREATE FUNCTION
+1
+1
+1
+1
+1
+1
+DROP FUNCTION
+denied_table2|t
+denied_tableas2|t
+denied_view2|t
+denied_mview2|t
+denied_selectinto2|t
+1
+denied_function2|t
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+DROP FUNCTION
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+CREATE FUNCTION
+1
+1
+1
+1
+1
+1
+DROP FUNCTION
+denied_table3|t
+denied_tableas3|t
+denied_view3|t
+denied_mview3|t
+denied_selectinto3|t
+1
+denied_function3|t
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+DROP FUNCTION
+
+CREATE TABLE
+INSERT 0 1
+SELECT 1
+CREATE VIEW
+SELECT 1
+SELECT 1
+CREATE FUNCTION
+1
+1
+1
+1
+1
+1
+denied_function4|t
+1
+1
+1
+1
+1
+1
+DROP TABLE
+DROP VIEW
+DROP MATERIALIZED VIEW
+DROP TABLE
+DROP TABLE
+DROP FUNCTION
+

test/CDB_QuotaTest.sql

@@ -1,30 +1,48 @@
 set client_min_messages to error;
 \set VERBOSITY TERSE
-
--- See the dice
-SELECT setseed(0.5);
+-- Runs a query and returns whether an error was thrown
+-- Useful when the error message depends on the execution plan or db settings
+-- The error message outputs the extra quota, and this might depend on the database setup and version
+CREATE OR REPLACE FUNCTION catch_error(query text)
+RETURNS bool
+AS $$
+BEGIN
+    EXECUTE query;
+    RETURN FALSE;
+EXCEPTION
+    WHEN OTHERS THEN
+        RETURN TRUE;
+END
+$$ LANGUAGE 'plpgsql';
 
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
-SELECT CDB_SetUserQuotaInBytes(0);
-SELECT CDB_CartodbfyTable('big');
+SELECT 'excess1', catch_error($$INSERT INTO big VALUES (2); $$); -- disallowed, quota exceeds before
+SELECT cartodb.CDB_SetUserQuotaInBytes(0);
+SELECT cartodb.CDB_CartodbfyTable('big');
+-- Creating the trigger should fail as it was created by CDB_CartodbfyTable
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+-- Drop the trigger and recreate it forcing a 100% checks
+DROP TRIGGER test_quota ON big;
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
 INSERT INTO big SELECT generate_series(2049,4096);
 INSERT INTO big SELECT generate_series(4097,6144);
 INSERT INTO big SELECT generate_series(6145,8192);
 -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
-SELECT CDB_UserDataSize();
-SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb.CDB_UserDataSize() < 500000 AND cartodb.CDB_UserDataSize() > 0;
+SELECT cartodb._CDB_total_relation_size('public', 'big') < 1000000;
 SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
 -- END Test for #108
-SELECT setseed(0.9);
-SELECT CDB_SetUserQuotaInBytes(2);
-INSERT INTO big VALUES (8193);
-SELECT CDB_SetUserQuotaInBytes(0);
+
+SELECT cartodb.CDB_SetUserQuotaInBytes(2);
+SELECT 'excess2', catch_error($$INSERT INTO big VALUES (8193);$$);
+SELECT cartodb.CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (8194);
 DROP TABLE big;
 
@@ -32,16 +50,17 @@ DROP TABLE big;
 --analysis tables should be excluded from quota:
 CREATE TABLE big(a int);
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
-SELECT CDB_SetUserQuotaInBytes(1);
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+SELECT cartodb.CDB_SetUserQuotaInBytes(1);
 CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int);
 INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5);
 INSERT INTO big VALUES (1); -- allowed, check runs before
 DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4;
-INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+SELECT 'excess3', catch_error($$INSERT INTO big VALUES (3);$$); -- disallowed, quota exceeds before
 DROP TABLE big;
 SELECT CDB_SetUserQuotaInBytes(0);
 
 
 set client_min_messages to NOTICE;
+DROP FUNCTION catch_error(text);
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,20 +1,22 @@
 SET
-
+CREATE FUNCTION
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
-ERROR:  Quota exceeded by 3.9990234375KB
+excess1|t
 0
 big
+ERROR:  trigger "test_quota" for relation "big" already exists
+DROP TRIGGER
+CREATE TRIGGER
 INSERT 0 2048
 INSERT 0 2048
 INSERT 0 2048
-454656
-909312
+t
+t
 0
-
 2
-ERROR:  Quota exceeded by 443.998046875KB
+excess2|t
 0
 INSERT 0 1
 DROP TABLE
@@ -25,8 +27,9 @@ CREATE TABLE
 INSERT 0 5
 INSERT 0 1
 DROP TABLE
-ERROR:  Quota exceeded by 3.9990234375KB
+excess3|t
 DROP TABLE
 0
 SET
 DROP FUNCTION
+DROP FUNCTION

test/CDB_UserTablesTest.sql

@@ -17,16 +17,16 @@ CREATE TABLE pub(a int);
 CREATE TABLE prv(a int);
 GRANT SELECT ON TABLE pub TO publicuser;
 REVOKE SELECT ON TABLE prv FROM publicuser;
-SELECT CDB_UserTables() ORDER BY 1;
-SELECT 'all',CDB_UserTables('all') ORDER BY 2;
-SELECT 'public',CDB_UserTables('public') ORDER BY 2;
-SELECT 'private',CDB_UserTables('private') ORDER BY 2;
-SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2;
+SELECT cartodb.CDB_UserTables() ORDER BY 1;
+SELECT 'all', cartodb.CDB_UserTables('all') ORDER BY 2;
+SELECT 'public', cartodb.CDB_UserTables('public') ORDER BY 2;
+SELECT 'private', cartodb.CDB_UserTables('private') ORDER BY 2;
+SELECT '--unsupported--', cartodb.CDB_UserTables('--unsupported--') ORDER BY 2;
 -- now tests with public user
 \c contrib_regression publicuser
-SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2;
-SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2;
-SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2;
+SELECT 'all_publicuser', cartodb.CDB_UserTables('all') ORDER BY 2;
+SELECT 'public_publicuser', cartodb.CDB_UserTables('public') ORDER BY 2;
+SELECT 'private_publicuser', cartodb.CDB_UserTables('private') ORDER BY 2;
 \c contrib_regression postgres
 DROP TABLE pub;
 DROP TABLE prv;

test/README

@@ -6,6 +6,7 @@ Example, to add a test for CDB_Something function, you'd add:
  - CDB_SomethingTest.sql
  - CDB_SomethingTest_expect
 
+
 To easy the generation of the expected file you can initially omit it,
 then run "make -C .. installcheck" from the top-level dir and copy
 ../results/test/CDB_SomethingTest.out to CDB_SomethingTest_expect chopping

test/extension/test.sh

@@ -17,6 +17,10 @@ SED=sed
 OK=0
 PARTIALOK=0
 
+function reset_default_database() {
+    DATABASE=test_extension
+}
+
 function set_failed() {
     OK=1
     PARTIALOK=1
@@ -40,10 +44,10 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      log_debug "Executing query '${QUERY}' as '${ROLE}' in '${DATABASE}'"
       RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
     else
-      log_debug "Executing query '${QUERY}'"
+      log_debug "Executing query '${QUERY}' in '${DATABASE}'"
       RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
     fi
     CODERESULT=$?
@@ -212,6 +216,8 @@ function tear_down_database() {
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
 function tear_down() {
+    reset_default_database
+
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
     sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
@@ -362,7 +368,7 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
 
 function test_cdb_tablemetadatatouch_fully_qualifies_names() {
     sql postgres "CREATE TABLE touch_invalidations (table_name text);"
-    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into public.touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
 
     #default schema
     sql "CREATE TABLE touch_example (a int);"
@@ -532,6 +538,8 @@ END
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
 
+    reset_default_database
+
     # Add PGPORT to conf if it is set
     PORT_SPEC=""
     if [[ "$PGPORT" != "" ]] ; then
@@ -590,6 +598,68 @@ test_extension|public|"local-table-with-dashes"'
     sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;"
     sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't'
 
+
+    # Check user-defined FDW's
+    # Set up a user foreign server
+    read -d '' ufdw_config <<- EOF
+{
+   "server": {
+     "extensions": "postgis",
+     "dbname": "fdw_target",
+     "host": "localhost",
+     "port": ${PGPORT:-5432}
+   },
+   "user_mapping": {
+     "user": "fdw_user",
+     "password": "foobarino"
+   }
+}
+EOF
+    sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');"
+
+    # Grant a user access to that FDW, and to grant to others
+    sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;'
+
+    # Set up a user foreign table
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');"
+
+    # Check that the table can be accessed by the owner/creator
+    sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;'
+    sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+
+    # Check that a role with no permissions cannot use the FDW to access a remote table
+    sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails
+
+    # Check that the table can be accessed by some other user by granting the role
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;'
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;'
+
+    # Check that the table can be accessed by org members
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');"
+    sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');"
+
+    # By default publicuser cannot access the FDW
+    sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
+    sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted
+    sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
+    sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;'
+
+    # If there are dependent objects, we cannot drop the foreign server
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails
+    sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;'
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')"
+
+    # But if there are, we can set the force flag to true to drop everything (defaults to false)
+    sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');"
+    sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;'
+    sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');"
+    sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)"
+
+
     # Teardown
     DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'
@@ -597,8 +667,10 @@ test_extension|public|"local-table-with-dashes"'
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
 
+    reset_default_database
     sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
     DATABASE=fdw_target tear_down_database
+    reset_default_database
 }
 
 function test_cdb_catalog_basic_node() {

util/create_from_unpackaged.sh

@@ -10,6 +10,7 @@ echo "-- Script generated by $0 on `date`" > ${output}
 cat ${input} |
   grep '^ *CREATE OR REPLACE FUNCTION' |
   grep -v ' cartodb\.' | # should  only match DDL hooks
+  grep -v '.*\quit.*' | 
   sed 's/).*$/)/' |
   sed 's/DEFAULT [^ ,)]*//g' |
   sed 's/CREATE OR REPLACE FUNCTION /ALTER FUNCTION public./' |
@@ -19,59 +20,4 @@ cat ${input} |
   cat >> ${output}
 
 # Upgrade all functions
-cat ${input} | grep -v 'duplicated extension$' >> ${output}
-
-# Migrate CDB_TableMetadata
-cat >> ${output} <<'EOF'
-ALTER TABLE cartodb.CDB_TableMetadata DISABLE TRIGGER ALL;
-INSERT INTO cartodb.CDB_TableMetadata SELECT * FROM public.CDB_TableMetadata;
-ALTER TABLE cartodb.CDB_TableMetadata ENABLE TRIGGER ALL;
-DROP TABLE public.CDB_TableMetadata;
-
--- Set user quota
--- NOTE: will fail if user quota wasn't set at database level, see
---       http://github.com/CartoDB/cartodb-postgresql/issues/18
-DO $$
-DECLARE
-  qmax int8;
-BEGIN
-  BEGIN
-    qmax := public._CDB_UserQuotaInBytes();
-  EXCEPTION WHEN undefined_function THEN
-    RAISE EXCEPTION 'Please set user quota before switching to cartodb extension';
-  END;
-  PERFORM cartodb.CDB_SetUserQuotaInBytes(qmax);
-  DROP FUNCTION public._CDB_UserQuotaInBytes();
-END;
-$$ LANGUAGE 'plpgsql';
-EOF
-
-## Cartodbfy tables with a trigger using 'CDB_CheckQuota' or
-## 'CDB_TableMetadata_Trigger' from the 'public' schema
-#cat >> ${output} <<'EOF'
-#select cartodb.CDB_CartodbfyTable(relname::regclass) from ( 
-#  -- names of tables using public.CDB_CheckQuota or
-#  -- public.CDB_TableMetadata_Trigger in their triggers
-#  SELECT distinct c.relname
-#  FROM
-#    pg_trigger t,
-#    pg_class c,
-#    pg_proc p,
-#    pg_namespace n
-#  WHERE
-#    n.nspname = 'public' AND
-#    p.pronamespace = n.oid AND
-#    p.proname IN ( 'cdb_checkquota', 'cdb_tablemetadata_trigger' ) AND
-#    t.tgrelid = c.oid AND
-#    p.oid = t.tgfoid 
-#) as foo;
-#EOF
-
-## Drop any leftover function from public schema (there should be none)
-#cat ${input} |
-#  grep '^ *CREATE OR REPLACE FUNCTION' |
-#  grep -v ' cartodb\.' | # should  only match DDL hooks
-#  sed 's/).*$/);/' |
-#  sed 's/DEFAULT [^ ,)]*//g' |
-#  sed 's/CREATE OR REPLACE FUNCTION /DROP FUNCTION IF EXISTS public./' |
-#  cat >> ${output}
+cat ${input} | grep -v 'duplicated extension$' | grep -v '\quit$' | grep -v 'pg_extension_config_dump' >> ${output}