Auxiliar build to revert to 0.32.0

Fix OAuth ownership re-assignation for functions

.travis.yml

@@ -8,26 +8,28 @@ env:
     - PGDATABASE=postgres
     - PGOPTIONS='-c client_min_messages=NOTICE'
     - PGPORT=5432
-    - POSTGIS_VERSION="2.5"
 
-  matrix:
-    - POSTGRESQL_VERSION="9.6"
-    - POSTGRESQL_VERSION="10"
-    - POSTGRESQL_VERSION="11"
+jobs:
+  include:
+    - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
+    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
+  
 
-
-before_install:
+script:
   - sudo service postgresql stop;
   - sudo apt-get remove postgresql* -y
   - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
   - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi;
-  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
+  # For pre12, install plpython2. For PG12 install plpython3
+  - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi;
   - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
   - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
   - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password
   - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
-  - sudo pip install redis==2.4.9
-script:
   - make
   - sudo make install
   - make installcheck

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.30.0
+EXTVERSION = 0.32.0
 
 SED = sed
 AWK = awk
@@ -103,6 +103,10 @@ UPGRADABLE = \
   0.28.1 \
   0.29.0 \
   0.30.0 \
+  0.31.0 \
+  0.32.0 \
+  0.33.0 \
+  0.34.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -126,19 +130,25 @@ EXTRA_CLEAN = cartodb_version.sql
 DOCS = README.md
 REGRESS_OLD = $(wildcard test/*.sql)
 REGRESS_LEGACY = $(REGRESS_OLD:.sql=)
-REGRESS = test_setup $(REGRESS_LEGACY)
+REGRESS = test/test_setup $(REGRESS_LEGACY)
 
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
 PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
 
 $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
 	$(SED) 	-e 's/@extschema@/cartodb/g' \
-		-e "s/@postgisschema@/public/g" >> $@
+		-e 's/@postgisschema@/public/g' \
+		-e 's/plpythonu/$(PLPYTHONU)/g' >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 
@@ -152,10 +162,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 	cp $< $@
 
 $(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
 
 # Needed for consistent `echo` results with backslashes
 SHELL = bash
@@ -164,6 +174,10 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
 	mkdir -p expected/test/
 	mkdir -p results/test/
+	cat sql/test_setup.sql | \
+			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' \
+			> sql/test/test_setup.sql
+	cp sql/test_setup_expect expected/test/test_setup.out
 	for f in $(REGRESS_OLD); do \
 		tn=`basename $${f} .sql`; \
 		of=sql/test/$${tn}.sql; \
@@ -172,14 +186,10 @@ legacy_regress: $(REGRESS_OLD) Makefile
 		echo '\t' >> $${of}; \
 		echo '\set QUIET off' >> $${of}; \
 		cat $${f} | \
-			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \
+			$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' >> $${of}; \
 		exp=expected/test/$${tn}.out; \
 		echo '\set ECHO none' > $${exp}; \
-		if [[ -f "test/$${tn}_expect.pg$(PG_VERSION)" ]]; then \
-			cat test/$${tn}_expect.pg$(PG_VERSION) >> $${exp}; \
-		else \
-			cat test/$${tn}_expect >> $${exp}; \
-		fi \
+		cat test/$${tn}_expect >> $${exp}; \
 	done
 
 test_organization:
@@ -188,7 +198,11 @@ test_organization:
 test_extension_new:
 	bash test/extension/test.sh
 
-legacy_tests: legacy_regress
+legacy_tests: legacy_regress $(EXTENSION)--unpackaged--$(EXTVERSION).sql
+
+PGREGRESS := $(shell dirname `$(PG_CONFIG) --pgxs`)/../../src/test/regress/pg_regress
+regress: legacy_tests
+	$(PGREGRESS) --inputdir=./ --bindir='/usr/bin'    --dbname=contrib_regression $(REGRESS)
 
 installcheck: legacy_tests test_extension_new test_organization
 

NEWS.md

@@ -1,3 +1,14 @@
+0.32.0 (2019-11-08)
+* Fix oAuth ownership re-assignation for functions
+* Some fixes for PG12.
+* Make PG12 depend on plpython3u instead of plpythonu
+* CDB_UserDataSize is now compatible with postgis 3 without postgis_raster.
+* Makefile: Add regress target (checks regress tests without needing to install the extension)
+
+0.31.0 (2019-10-08)
+* Ghost tables: Add missing tags (#370)
+* Set search_path in security definer functions.
+
 0.30.0 (2019-07-17)
 * Added new admin functions to connect CARTO with user FDW's (#369)
 

README.md

@@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/
 Dependencies
 ------------
 
- * PostgreSQL 9.6+ (with plpythonu extension and xml support)
+ * PostgreSQL 9.6+ (with plpythonu extension and xml support). For PostgreSQL 12+ plpython3u is required instead of plpythonu.
  * [PostGIS extension](http://postgis.net)
  * Python with [Redis module](https://pypi.org/project/redis/)
 

expected/test_setup.out

@@ -1,6 +1,5 @@
-CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
+SET client_min_messages TO error;
+CREATE EXTENSION cartodb CASCADE;
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN

scripts-available/CDB_AnalysisCheck.sql

@@ -6,8 +6,11 @@ $$
 BEGIN
   RETURN @extschema@.CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
 END;
-$$
-LANGUAGE 'plpgsql' STABLE PARALLEL SAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 
 -- Get the factor (fraction of the quota) for Camshaft cached analysis tables

scripts-available/CDB_EstimateRowCount.sql

@@ -12,7 +12,12 @@ BEGIN
     EXECUTE Format('ANALYZE %s;', reloid);
   END IF;
 END
-$$ LANGUAGE 'plpgsql' VOLATILE STRICT PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpgsql'
+    VOLATILE
+    STRICT
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Return a row count estimate of the result of a query using statistics
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EstimateRowCount(query text)

scripts-available/CDB_ForeignTable.sql

@@ -191,13 +191,13 @@ BEGIN
   -- (not even using IF NOT EXIST to avoid throwing warnings)
   IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
     CREATE EXTENSION postgres_fdw;
-    RAISE NOTICE 'Created postgres_fdw extension';
+    RAISE NOTICE 'Created postgres_fdw EXTENSION';
   END IF;
   -- Create FDW first if it does not exist
   IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name)
     THEN
     EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name);
-    RAISE NOTICE 'Created server % using postgres_fdw', fdw_objects_name;
+    RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name;
   END IF;
 
   -- Set FDW settings
@@ -214,7 +214,7 @@ BEGIN
     -- Create specific role for this
     IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN
        EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name);
-       RAISE NOTICE 'Created special role % to access the correponding FDW', fdw_objects_name;
+       RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name;
     END IF;
 
     -- Transfer ownership of the server to the fdw role
@@ -225,7 +225,7 @@ BEGIN
     -- so that we don't need to create a mapping for every user nor store credentials elsewhere
     IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN
         EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name);
-        RAISE NOTICE 'Created user mapping for accesing foreign server %', fdw_objects_name;
+        RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name;
     END IF;
 
     -- Update user mapping settings
@@ -239,19 +239,19 @@ BEGIN
 
     -- Grant usage on the wrapper and server to the fdw role
     EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name);
-    RAISE NOTICE 'Granted usage on the postgres_fdw to the role %', fdw_objects_name;
+    RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name;
     EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name);
-    RAISE NOTICE 'Granted usage on the foreign server to the role %', fdw_objects_name;
+    RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name;
 
     -- Create schema if it does not exist.
     IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN
       EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name);
-      RAISE NOTICE 'Created schema % to host foreign tables', fdw_objects_name;
+      RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name;
     END IF;
 
     -- Give the fdw role ownership over the schema
     EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
-    RAISE NOTICE 'Gave ownership on the schema % to %', fdw_objects_name, fdw_objects_name;
+    RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name;
 
     -- TODO: Bring here the remote cdb_tablemetadata
 END

scripts-available/CDB_GhostTables.sql

@@ -63,7 +63,11 @@ AS $$
     PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
     RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Trigger function to call CDB_LinkGhostTables()
 CREATE OR REPLACE FUNCTION @extschema@._CDB_LinkGhostTablesTrigger()
@@ -76,7 +80,11 @@ AS $$
     PERFORM @extschema@.CDB_LinkGhostTables(ddl_tag);
     RETURN NULL;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Event trigger to save the current transaction in @extschema@.cdb_ddl_execution
 CREATE OR REPLACE FUNCTION @extschema@.CDB_SaveDDLTransaction()
@@ -85,7 +93,11 @@ AS $$
   BEGIN
     INSERT INTO @extschema@.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT ON CONSTRAINT cdb_ddl_execution_pkey DO NOTHING;
   END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Creates the trigger on DDL events to link ghost tables
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableGhostTablesTrigger()
@@ -106,7 +118,33 @@ AS $$
 
     CREATE EVENT TRIGGER link_ghost_tables
     ON ddl_command_end
-    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW', 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE', 'DROP FOREIGN TABLE')
+    WHEN TAG IN ('CREATE TABLE',
+                'SELECT INTO',
+                'DROP TABLE',
+                'ALTER TABLE',
+
+                'CREATE TRIGGER',
+                'DROP TRIGGER',
+                'ALTER TRIGGER',
+
+                'CREATE VIEW',
+                'DROP VIEW',
+                'ALTER VIEW',
+
+                'CREATE FOREIGN TABLE',
+                'ALTER FOREIGN TABLE',
+                'DROP FOREIGN TABLE',
+
+                'ALTER MATERIALIZED VIEW',
+                'CREATE MATERIALIZED VIEW',
+                'DROP MATERIALIZED VIEW',
+
+                'IMPORT FOREIGN SCHEMA',
+
+                'DROP EXTENSION',
+                'DROP SCHEMA',
+                'DROP SERVER',
+                'DROP TYPE')
     EXECUTE PROCEDURE @extschema@.CDB_SaveDDLTransaction();
   END;
 $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_Groups_API.sql

@@ -22,59 +22,91 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text)
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(group_name))
 
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(old_group_name))
     body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name))
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -89,13 +121,20 @@ FUNCTION @extschema@._CDB_Group_Table_GrantPermission_API(group_name text, usern
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
     body = '{ "access": "%s" }' % access
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -110,12 +149,19 @@ FUNCTION @extschema@._CDB_Group_Table_RevokeAllPermission_API(group_name text, u
     RETURNS VOID AS
 $$
     import string
-    import urllib
+    try:
+        from urllib import pathname2url
+    except:
+        from urllib.request import pathname2url
 
-    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
     plpy.execute(query)
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE 'plpythonu'
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DO LANGUAGE 'plpgsql' $$
 BEGIN
@@ -161,7 +207,10 @@ CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
     RETURNS int AS
 $$
-    import httplib
+    try:
+        import httplib as client
+    except:
+        from http import client
 
     params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0]
     if params['host'] is None:
@@ -174,17 +223,17 @@ $$
     last_err = None
     while retry > 0:
       try:
-        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout'])
         database_name = plpy.execute("select current_database();")[0]['current_database']
-        client.request(method, url.format(database_name), body, headers)
-        response = client.getresponse()
+        conn.request(method, url.format(database_name), body, headers)
+        response = conn.getresponse()
         assert response.status in valid_return_codes
         return response.status
       except Exception as err:
         retry -= 1
         last_err = err
         plpy.warning('Retrying after: ' + str(err))
-        client = SD['groups_api_client'] = None
+        conn = SD['groups_api_client'] = None
 
     if last_err is not None:
       plpy.error('Fatal Group API error: ' + str(last_err))

scripts-available/CDB_OAuth.sql

@@ -1,7 +1,6 @@
 -- Function that reassign the owner of a table to their ownership_role
 CREATE OR REPLACE FUNCTION @extschema@.CDB_OAuthReassignTableOwnerOnCreation()
   RETURNS event_trigger
-  SECURITY DEFINER
   AS $$
 DECLARE
     obj record;
@@ -16,18 +15,31 @@ BEGIN
                   obj.object_type,
                   obj.schema_name,
                   obj.object_identity;
-      SELECT rolname FROM pg_class JOIN pg_roles ON relowner = pg_roles.oid WHERE pg_class.oid = obj.objid INTO creator_role;
-      SELECT value->>'ownership_role_name' from cdb_conf where key = 'api_keys_' || creator_role INTO owner_role;
+      IF obj.object_type = 'function' THEN
+        SELECT rolname FROM pg_proc JOIN pg_roles ON proowner = pg_roles.oid WHERE pg_proc.oid = obj.objid INTO creator_role;
+      ELSE
+	      SELECT rolname FROM pg_class JOIN pg_roles ON relowner = pg_roles.oid WHERE pg_class.oid = obj.objid INTO creator_role;
+      END IF;
+      SELECT value->>'ownership_role_name' from @extschema@.CDB_Conf_GetConf('api_keys_' || quote_ident(creator_role)) value INTO owner_role;
       IF owner_role IS NULL OR owner_role = '' THEN
+        RAISE DEBUG 'owner_role not found';
         CONTINUE;
       ELSE
-        EXECUTE 'ALTER ' || obj.object_type || ' ' || obj.object_identity || ' OWNER TO ' || QUOTE_IDENT(owner_role);
-        EXECUTE 'GRANT ALL ON ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        EXECUTE 'ALTER ' || obj.object_type || ' ' || obj.object_identity || ' OWNER TO ' || quote_ident(owner_role);
+        IF obj.object_type = 'function' THEN
+          EXECUTE 'GRANT ALL ON FUNCTION ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        ELSE
+          EXECUTE 'GRANT ALL ON ' || obj.object_identity || ' TO ' || QUOTE_IDENT(creator_role);
+        END IF;        
         RAISE DEBUG 'Changing ownership from % to %', creator_role, owner_role;
       END IF;
     END LOOP;
 END;
-$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Creates the trigger on DDL events in order to reassign the owner
 CREATE OR REPLACE FUNCTION @extschema@.CDB_EnableOAuthReassignTablesTrigger()

scripts-available/CDB_Overviews.sql

@@ -318,7 +318,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
 -- This function is declared SECURITY DEFINER so it executes with the privileges
 -- of the function creator to have a chance to alter the privileges of the
 -- overview table to match those of the dataset. It will only perform any change
--- if the overview table belgons to the same scheme as the dataset and it
+-- if the overview table belongs to the same scheme as the dataset and it
 -- matches the scheme naming for overview tables.
 CREATE OR REPLACE FUNCTION @extschema@._CDB_Register_Overview(dataset REGCLASS, overview_table REGCLASS, overview_z INTEGER)
 RETURNS VOID
@@ -362,7 +362,11 @@ AS $$
       -- it should be done here (CDB_Overviews would consume such metadata)
     END IF;
   END
-$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE PLPGSQL
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 -- Dataset attributes (column names other than the
 -- CartoDB primary key and geometry columns) which should be aggregated
@@ -393,7 +397,7 @@ DECLARE
   attr_list TEXT;
 BEGIN
   SELECT string_agg(s.c, ',') FROM (
-    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
+    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -550,7 +554,7 @@ DECLARE
 BEGIN
   SELECT string_agg(@extschema@._CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',')
   FROM (
-    SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c
+    SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c
   ) AS s INTO attr_list;
 
   RETURN attr_list;
@@ -658,7 +662,7 @@ AS $$
       offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
     END IF;
 
-    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+    point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
 
     -- compute the resulting columns in the same order as in the base table
     WITH cols AS (
@@ -669,7 +673,7 @@ AS $$
           Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom)
         WHEN 'the_geom_webmercator' THEN
            Format('%s AS the_geom_webmercator', point_geom)
-        ELSE c
+        ELSE c::text
         END AS column
         FROM @extschema@.CDB_ColumnNames(reloid) c
     )
@@ -796,7 +800,7 @@ AS $$
           '@postgisschema@.ST_Transform(@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom'
         WHEN 'the_geom_webmercator' THEN
           '@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator'
-        ELSE c
+        ELSE c::text
         END AS column
         FROM CDB_ColumnNames(reloid) c
     )
@@ -920,7 +924,7 @@ AS $$
       SELECT
         CASE c
         WHEN 'cartodb_id' THEN 'cartodb_id'
-        ELSE c
+        ELSE c::text
         END AS column
         FROM @extschema@.CDB_ColumnNames(reloid) c
     )

scripts-available/CDB_Quota.sql

@@ -20,33 +20,50 @@ RETURNS bigint AS
 $$
 DECLARE
   total_size INT8;
+  raster_available BOOLEAN;
+  raster_read_query TEXT;
 BEGIN
+  -- Postgis 3+ might not install raster
+  raster_available := EXISTS (
+        SELECT 1
+        FROM   pg_views
+        WHERE  schemaname = '@postgisschema@'
+        AND    viewname = 'raster_overviews'
+   );
+
+   IF raster_available THEN
+       raster_read_query := Format('SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews
+                        WHERE o_table_schema = ''%I'' AND o_table_catalog = current_database()', schema_name);
+   ELSE
+       raster_read_query := 'SELECT NULL::text AS o_table_name, NULL::text AS r_table_name';
+   END IF;
+   EXECUTE Format('
   WITH raster_tables AS (
-    SELECT o_table_name, r_table_name FROM raster_overviews
-      WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
+    %s
   ),
   user_tables AS (
-    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(schema_name)
+    SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(''%I'')
   ),
   table_cat AS (
     SELECT
       table_name,
       (
         EXISTS(select * from raster_tables where o_table_name = table_name)
-        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || '[\w\d]*'
+        OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || ''[\w\d]*''
       ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),
   sizes AS (
-    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(schema_name, table_name)))) table_size,
+    SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(''%I'', table_name)))) table_size,
       CASE
         WHEN is_overview THEN 0
 	WHEN is_raster THEN 1
 	ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator
       END AS multiplier FROM table_cat GROUP BY is_overview, is_raster
   )
-  SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes;
+  SELECT sum(table_size*multiplier)::int8 FROM sizes
+  ', raster_read_query, schema_name, schema_name) INTO total_size;
 
   IF total_size IS NOT NULL THEN
     RETURN total_size;

scripts-available/CDB_TableMetadata.sql

@@ -43,7 +43,7 @@ BEGIN
   );
 
   WITH nv as (
-    SELECT TG_RELID as tabname, NOW() as t
+    SELECT TG_RELID as tabname, now() as t
   ), updated as (
     UPDATE @extschema@.CDB_TableMetadata x SET updated_at = nv.t
     FROM nv WHERE x.tabname = nv.tabname
@@ -55,8 +55,11 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 --
 -- Trigger invalidating varnish whenever CDB_TableMetadata
@@ -116,8 +119,11 @@ BEGIN
 
   RETURN NULL;
 END;
-$$
-LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+$$  LANGUAGE plpgsql
+    VOLATILE
+    PARALLEL UNSAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;
 
 DROP TRIGGER IF EXISTS table_modified ON @extschema@.CDB_TableMetadata;
 -- NOTE: on DELETE we would be unable to convert the table

scripts-available/CDB_Username.sql

@@ -2,5 +2,9 @@
 CREATE OR REPLACE FUNCTION @extschema@.CDB_Username()
 RETURNS text
 AS $$
-  SELECT @extschema@.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
-$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;
+  SELECT @extschema@.CDB_Conf_GetConf(concat('api_keys_', session_user))->>'username';
+$$  LANGUAGE SQL
+    STABLE
+    PARALLEL SAFE
+    SECURITY DEFINER
+    SET search_path = pg_temp;

sql/test_setup.sql

@@ -1,9 +1,14 @@
+\set ECHO none
+\set QUIET on
+SET client_min_messages TO error;
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
+CREATE SCHEMA cartodb;
+\i 'cartodb--unpackaged--@@VERSION@@.sql'
 CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)
 RETURNS void AS $$
 BEGIN
   RAISE NOTICE 'cdb_invalidate_varnish(%) called', table_name;
 END;
 $$ LANGUAGE 'plpgsql';
+\set QUIET off

sql/test_setup_expect

@@ -0,0 +1 @@
+\set ECHO none

test/CDB_GhostTables_expect

@@ -4,7 +4,7 @@ WARNING:  Invalidation service configuration not found. Skipping Ghost Tables li
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 BEGIN
@@ -12,7 +12,7 @@ cdb_ddl_execution
 0
 CREATE TABLE
 1
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
 NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
 COMMIT
 

test/CDB_OAuth.sql

@@ -1,6 +1,23 @@
 -- Create user and enable OAuth event trigger
 \set QUIET on
 SET client_min_messages TO error;
+
+-- The permission error changed between pre PG11 and post 11 (before everything was "relation", now it's "view", "table" and so on
+CREATE OR REPLACE FUNCTION catch_permission_error(query text)
+    RETURNS bool
+AS $$
+BEGIN
+    EXECUTE query;
+    RETURN FALSE;
+EXCEPTION
+    WHEN insufficient_privilege THEN
+        RETURN TRUE;
+    WHEN OTHERS THEN
+        RAISE WARNING 'Exception %', sqlstate;
+        RETURN FALSE;
+END
+$$ LANGUAGE 'plpgsql';
+
 DROP ROLE IF EXISTS "creator_role";
 CREATE ROLE "creator_role" LOGIN;
 DROP ROLE IF EXISTS "ownership_role";
@@ -19,22 +36,30 @@ CREATE TABLE test_tablesas AS SELECT * FROM test;
 CREATE VIEW test_view AS SELECT * FROM test;
 CREATE MATERIALIZED VIEW test_mview AS SELECT * FROM test;
 SELECT * INTO test_selectinto FROM test;
+CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 
 SELECT * FROM test;
 SELECT * FROM test_tablesas;
 SELECT * FROM test_view;
 SELECT * FROM test_mview;
 SELECT * FROM test_selectinto;
+SELECT test_function();
+-- Postgres grants default execute privilege on functions to PUBLIC. So in order to check the different permissions
+-- between creator and owner roles is not enough with performing a selection, we need to DROP the table (which only the owner can do)
+DROP FUNCTION test_function();
 
 \set QUIET on
+CREATE FUNCTION test_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 SET SESSION AUTHORIZATION "ownership_role";
 \set QUIET off
 
-SELECT * FROM test;
-SELECT * FROM test_tablesas;
-SELECT * FROM test_view;
-SELECT * FROM test_mview;
-SELECT * FROM test_selectinto;
+SELECT 'denied_table', catch_permission_error($$SELECT * FROM test;$$);
+SELECT 'denied_tableas', catch_permission_error($$SELECT * FROM test_tablesas;$$);
+SELECT 'denied_view', catch_permission_error($$SELECT * FROM test_view;$$);
+SELECT 'denied_mview', catch_permission_error($$SELECT * FROM test_mview;$$);
+SELECT 'denied_selectinto', catch_permission_error($$SELECT * FROM test_selectinto;$$);
+SELECT test_function();
+SELECT 'denied_function', catch_permission_error($$DROP FUNCTION test_function();$$);
 
 \set QUIET on
 SET SESSION AUTHORIZATION "creator_role";
@@ -45,6 +70,7 @@ DROP VIEW test_view;
 DROP MATERIALIZED VIEW test_mview;
 DROP TABLE test_selectinto;
 DROP TABLE test;
+DROP FUNCTION test_function();
 
 -- Second part with event trigger but without ownership_role_name in cdb_conf
 
@@ -60,22 +86,28 @@ CREATE TABLE test2_tablesas AS SELECT * FROM test2;
 CREATE VIEW test2_view AS SELECT * FROM test2;
 CREATE MATERIALIZED VIEW test2_mview AS SELECT * FROM test2;
 SELECT * INTO test2_selectinto FROM test2;
+CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 
 SELECT * FROM test2;
 SELECT * FROM test2_tablesas;
 SELECT * FROM test2_view;
 SELECT * FROM test2_mview;
 SELECT * FROM test2_selectinto;
+SELECT test2_function();
+DROP FUNCTION test2_function();
 
 \set QUIET on
+CREATE FUNCTION test2_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 SET SESSION AUTHORIZATION "ownership_role";
 \set QUIET off
 
-SELECT * FROM test2;
-SELECT * FROM test2_tablesas;
-SELECT * FROM test2_view;
-SELECT * FROM test2_mview;
-SELECT * FROM test2_selectinto;
+SELECT 'denied_table2', catch_permission_error($$SELECT * FROM test2;$$);
+SELECT 'denied_tableas2', catch_permission_error($$SELECT * FROM test2_tablesas;$$);
+SELECT 'denied_view2', catch_permission_error($$SELECT * FROM test2_view;$$);
+SELECT 'denied_mview2', catch_permission_error($$SELECT * FROM test2_mview;$$);
+SELECT 'denied_selectinto2', catch_permission_error($$SELECT * FROM test2_selectinto;$$);
+SELECT test2_function();
+SELECT 'denied_function2', catch_permission_error($$DROP FUNCTION test2_function();$$);
 
 \set QUIET on
 SET SESSION AUTHORIZATION "creator_role";
@@ -86,6 +118,7 @@ DROP VIEW test2_view;
 DROP MATERIALIZED VIEW test2_mview;
 DROP TABLE test2_selectinto;
 DROP TABLE test2;
+DROP FUNCTION test2_function();
 
 -- Third part with event trigger but with empty ownership_role_name in cdb_conf
 
@@ -101,22 +134,28 @@ CREATE TABLE test3_tablesas AS SELECT * FROM test3;
 CREATE VIEW test3_view AS SELECT * FROM test3;
 CREATE MATERIALIZED VIEW test3_mview AS SELECT * FROM test3;
 SELECT * INTO test3_selectinto FROM test3;
+CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 
 SELECT * FROM test3;
 SELECT * FROM test3_tablesas;
 SELECT * FROM test3_view;
 SELECT * FROM test3_mview;
 SELECT * FROM test3_selectinto;
+SELECT test3_function();
+DROP FUNCTION test3_function();
 
 \set QUIET on
+CREATE FUNCTION test3_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 SET SESSION AUTHORIZATION "ownership_role";
 \set QUIET off
 
-SELECT * FROM test3;
-SELECT * FROM test3_tablesas;
-SELECT * FROM test3_view;
-SELECT * FROM test3_mview;
-SELECT * FROM test3_selectinto;
+SELECT 'denied_table3', catch_permission_error($$SELECT * FROM test3;$$);
+SELECT 'denied_tableas3', catch_permission_error($$SELECT * FROM test3_tablesas;$$);
+SELECT 'denied_view3', catch_permission_error($$SELECT * FROM test3_view;$$);
+SELECT 'denied_mview3', catch_permission_error($$SELECT * FROM test3_mview;$$);
+SELECT 'denied_selectinto3', catch_permission_error($$SELECT * FROM test3_selectinto;$$);
+SELECT test3_function();
+SELECT 'denied_function3', catch_permission_error($$DROP FUNCTION test3_function();$$);
 
 \set QUIET on
 SET SESSION AUTHORIZATION "creator_role";
@@ -127,6 +166,7 @@ DROP VIEW test3_view;
 DROP MATERIALIZED VIEW test3_mview;
 DROP TABLE test3_selectinto;
 DROP TABLE test3;
+DROP FUNCTION test3_function();
 
 -- Fourth part with the event trigger active and configured
 
@@ -142,12 +182,15 @@ CREATE TABLE test4_tablesas AS SELECT * FROM test4;
 CREATE VIEW test4_view AS SELECT * FROM test4;
 CREATE MATERIALIZED VIEW test4_mview AS SELECT * FROM test4;
 SELECT * INTO test4_selectinto FROM test4;
+CREATE FUNCTION test4_function() RETURNS integer AS $$ BEGIN RETURN 1; END; $$ LANGUAGE PLPGSQL;
 
 SELECT * FROM test4;
 SELECT * FROM test4_tablesas;
 SELECT * FROM test4_view;
 SELECT * FROM test4_mview;
 SELECT * FROM test4_selectinto;
+SELECT test4_function();
+SELECT 'denied_function4', catch_permission_error($$DROP FUNCTION test4_function();$$);
 
 \set QUIET on
 SET SESSION AUTHORIZATION "ownership_role";
@@ -158,6 +201,7 @@ SELECT * FROM test4_tablesas;
 SELECT * FROM test4_view;
 SELECT * FROM test4_mview;
 SELECT * FROM test4_selectinto;
+SELECT test4_function();
 
 -- Ownership role drops the tables
 DROP TABLE test4_tablesas;
@@ -165,6 +209,7 @@ DROP VIEW test4_view;
 DROP MATERIALIZED VIEW test4_mview;
 DROP TABLE test4_selectinto;
 DROP TABLE test4;
+DROP FUNCTION test4_function();
 
 -- Cleanup
 \set QUIET on
@@ -174,4 +219,5 @@ DROP ROLE "ownership_role";
 REVOKE ALL ON SCHEMA cartodb FROM "creator_role";
 DROP ROLE "creator_role";
 DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role';
+DROP FUNCTION catch_permission_error(text);
 \set QUIET off

test/CDB_OAuth_expect

@@ -5,21 +5,27 @@ SELECT 1
 CREATE VIEW
 SELECT 1
 SELECT 1
+CREATE FUNCTION
 1
 1
 1
 1
 1
-ERROR:  permission denied for relation test
-ERROR:  permission denied for relation test_tablesas
-ERROR:  permission denied for relation test_view
-ERROR:  permission denied for relation test_mview
-ERROR:  permission denied for relation test_selectinto
+1
+DROP FUNCTION
+denied_table|t
+denied_tableas|t
+denied_view|t
+denied_mview|t
+denied_selectinto|t
+1
+denied_function|t
 DROP TABLE
 DROP VIEW
 DROP MATERIALIZED VIEW
 DROP TABLE
 DROP TABLE
+DROP FUNCTION
 NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
 
 CREATE TABLE
@@ -28,21 +34,27 @@ SELECT 1
 CREATE VIEW
 SELECT 1
 SELECT 1
+CREATE FUNCTION
 1
 1
 1
 1
 1
-ERROR:  permission denied for relation test2
-ERROR:  permission denied for relation test2_tablesas
-ERROR:  permission denied for relation test2_view
-ERROR:  permission denied for relation test2_mview
-ERROR:  permission denied for relation test2_selectinto
+1
+DROP FUNCTION
+denied_table2|t
+denied_tableas2|t
+denied_view2|t
+denied_mview2|t
+denied_selectinto2|t
+1
+denied_function2|t
 DROP TABLE
 DROP VIEW
 DROP MATERIALIZED VIEW
 DROP TABLE
 DROP TABLE
+DROP FUNCTION
 
 CREATE TABLE
 INSERT 0 1
@@ -50,21 +62,27 @@ SELECT 1
 CREATE VIEW
 SELECT 1
 SELECT 1
+CREATE FUNCTION
 1
 1
 1
 1
 1
-ERROR:  permission denied for relation test3
-ERROR:  permission denied for relation test3_tablesas
-ERROR:  permission denied for relation test3_view
-ERROR:  permission denied for relation test3_mview
-ERROR:  permission denied for relation test3_selectinto
+1
+DROP FUNCTION
+denied_table3|t
+denied_tableas3|t
+denied_view3|t
+denied_mview3|t
+denied_selectinto3|t
+1
+denied_function3|t
 DROP TABLE
 DROP VIEW
 DROP MATERIALIZED VIEW
 DROP TABLE
 DROP TABLE
+DROP FUNCTION
 
 CREATE TABLE
 INSERT 0 1
@@ -72,12 +90,16 @@ SELECT 1
 CREATE VIEW
 SELECT 1
 SELECT 1
+CREATE FUNCTION
 1
 1
 1
 1
 1
 1
+denied_function4|t
+1
+1
 1
 1
 1
@@ -87,4 +109,5 @@ DROP VIEW
 DROP MATERIALIZED VIEW
 DROP TABLE
 DROP TABLE
+DROP FUNCTION
 

test/CDB_OAuth_expect.pg11

@@ -1,90 +0,0 @@
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-1
-1
-1
-1
-1
-ERROR:  permission denied for table test
-ERROR:  permission denied for table test_tablesas
-ERROR:  permission denied for view test_view
-ERROR:  permission denied for materialized view test_mview
-ERROR:  permission denied for table test_selectinto
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-NOTICE:  event trigger "oauth_reassign_tables_trigger" does not exist, skipping
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-1
-1
-1
-1
-1
-ERROR:  permission denied for table test2
-ERROR:  permission denied for table test2_tablesas
-ERROR:  permission denied for view test2_view
-ERROR:  permission denied for materialized view test2_mview
-ERROR:  permission denied for table test2_selectinto
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-1
-1
-1
-1
-1
-ERROR:  permission denied for table test3
-ERROR:  permission denied for table test3_tablesas
-ERROR:  permission denied for view test3_view
-ERROR:  permission denied for materialized view test3_mview
-ERROR:  permission denied for table test3_selectinto
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-
-CREATE TABLE
-INSERT 0 1
-SELECT 1
-CREATE VIEW
-SELECT 1
-SELECT 1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-DROP TABLE
-DROP VIEW
-DROP MATERIALIZED VIEW
-DROP TABLE
-DROP TABLE
-

test/CDB_QuotaTest.sql

@@ -1,30 +1,48 @@
 set client_min_messages to error;
 \set VERBOSITY TERSE
-
--- See the dice
-SELECT setseed(0.5);
+-- Runs a query and returns whether an error was thrown
+-- Useful when the error message depends on the execution plan or db settings
+-- The error message outputs the extra quota, and this might depend on the database setup and version
+CREATE OR REPLACE FUNCTION catch_error(query text)
+RETURNS bool
+AS $$
+BEGIN
+    EXECUTE query;
+    RETURN FALSE;
+EXCEPTION
+    WHEN OTHERS THEN
+        RETURN TRUE;
+END
+$$ LANGUAGE 'plpgsql';
 
 CREATE TABLE big(a int);
 -- Try the legacy interface
 -- See https://github.com/CartoDB/cartodb-postgresql/issues/13
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
 INSERT INTO big VALUES (1); -- allowed, check runs before
-INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
-SELECT CDB_SetUserQuotaInBytes(0);
-SELECT CDB_CartodbfyTable('big');
+SELECT 'excess1', catch_error($$INSERT INTO big VALUES (2); $$); -- disallowed, quota exceeds before
+SELECT cartodb.CDB_SetUserQuotaInBytes(0);
+SELECT cartodb.CDB_CartodbfyTable('big');
+-- Creating the trigger should fail as it was created by CDB_CartodbfyTable
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+-- Drop the trigger and recreate it forcing a 100% checks
+DROP TRIGGER test_quota ON big;
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
 INSERT INTO big SELECT generate_series(2049,4096);
 INSERT INTO big SELECT generate_series(4097,6144);
 INSERT INTO big SELECT generate_series(6145,8192);
 -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108
-SELECT CDB_UserDataSize();
-SELECT cartodb._CDB_total_relation_size('public', 'big');
+SELECT cartodb.CDB_UserDataSize() < 500000 AND cartodb.CDB_UserDataSize() > 0;
+SELECT cartodb._CDB_total_relation_size('public', 'big') < 1000000;
 SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name');
 -- END Test for #108
-SELECT setseed(0.9);
-SELECT CDB_SetUserQuotaInBytes(2);
-INSERT INTO big VALUES (8193);
-SELECT CDB_SetUserQuotaInBytes(0);
+
+SELECT cartodb.CDB_SetUserQuotaInBytes(2);
+SELECT 'excess2', catch_error($$INSERT INTO big VALUES (8193);$$);
+SELECT cartodb.CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (8194);
 DROP TABLE big;
 
@@ -32,16 +50,17 @@ DROP TABLE big;
 --analysis tables should be excluded from quota:
 CREATE TABLE big(a int);
 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
-      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
-SELECT CDB_SetUserQuotaInBytes(1);
+      EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public');
+SELECT cartodb.CDB_SetUserQuotaInBytes(1);
 CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int);
 INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5);
 INSERT INTO big VALUES (1); -- allowed, check runs before
 DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4;
-INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+SELECT 'excess3', catch_error($$INSERT INTO big VALUES (3);$$); -- disallowed, quota exceeds before
 DROP TABLE big;
 SELECT CDB_SetUserQuotaInBytes(0);
 
 
 set client_min_messages to NOTICE;
+DROP FUNCTION catch_error(text);
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -1,20 +1,22 @@
 SET
-
+CREATE FUNCTION
 CREATE TABLE
 CREATE TRIGGER
 INSERT 0 1
-ERROR:  Quota exceeded by 3.9990234375KB
+excess1|t
 0
 big
+ERROR:  trigger "test_quota" for relation "big" already exists
+DROP TRIGGER
+CREATE TRIGGER
 INSERT 0 2048
 INSERT 0 2048
 INSERT 0 2048
-454656
-909312
+t
+t
 0
-
 2
-ERROR:  Quota exceeded by 443.998046875KB
+excess2|t
 0
 INSERT 0 1
 DROP TABLE
@@ -25,8 +27,9 @@ CREATE TABLE
 INSERT 0 5
 INSERT 0 1
 DROP TABLE
-ERROR:  Quota exceeded by 3.9990234375KB
+excess3|t
 DROP TABLE
 0
 SET
 DROP FUNCTION
+DROP FUNCTION

test/CDB_UserTablesTest.sql

@@ -17,16 +17,16 @@ CREATE TABLE pub(a int);
 CREATE TABLE prv(a int);
 GRANT SELECT ON TABLE pub TO publicuser;
 REVOKE SELECT ON TABLE prv FROM publicuser;
-SELECT CDB_UserTables() ORDER BY 1;
-SELECT 'all',CDB_UserTables('all') ORDER BY 2;
-SELECT 'public',CDB_UserTables('public') ORDER BY 2;
-SELECT 'private',CDB_UserTables('private') ORDER BY 2;
-SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2;
+SELECT cartodb.CDB_UserTables() ORDER BY 1;
+SELECT 'all', cartodb.CDB_UserTables('all') ORDER BY 2;
+SELECT 'public', cartodb.CDB_UserTables('public') ORDER BY 2;
+SELECT 'private', cartodb.CDB_UserTables('private') ORDER BY 2;
+SELECT '--unsupported--', cartodb.CDB_UserTables('--unsupported--') ORDER BY 2;
 -- now tests with public user
 \c contrib_regression publicuser
-SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2;
-SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2;
-SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2;
+SELECT 'all_publicuser', cartodb.CDB_UserTables('all') ORDER BY 2;
+SELECT 'public_publicuser', cartodb.CDB_UserTables('public') ORDER BY 2;
+SELECT 'private_publicuser', cartodb.CDB_UserTables('private') ORDER BY 2;
 \c contrib_regression postgres
 DROP TABLE pub;
 DROP TABLE prv;

test/README

@@ -6,13 +6,6 @@ Example, to add a test for CDB_Something function, you'd add:
  - CDB_SomethingTest.sql
  - CDB_SomethingTest_expect
 
-In case you need multiple expects of a test for different versions you have
-to add .pg$(VERSION) at the end of the file.
-
-For example if you want an expect file for PG11 you need to have two expect files:
-
-  - CDB_SomethingTest_expect
-  - CDB_SomethingTest_expect.pg11
 
 To easy the generation of the expected file you can initially omit it,
 then run "make -C .. installcheck" from the top-level dir and copy

test/extension/test.sh

@@ -17,6 +17,10 @@ SED=sed
 OK=0
 PARTIALOK=0
 
+function reset_default_database() {
+    DATABASE=test_extension
+}
+
 function set_failed() {
     OK=1
     PARTIALOK=1
@@ -40,10 +44,10 @@ function sql() {
     fi
 
     if [ -n "${ROLE}" ]; then
-      log_debug "Executing query '${QUERY}' as ${ROLE}"
+      log_debug "Executing query '${QUERY}' as '${ROLE}' in '${DATABASE}'"
       RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
     else
-      log_debug "Executing query '${QUERY}'"
+      log_debug "Executing query '${QUERY}' in '${DATABASE}'"
       RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
     fi
     CODERESULT=$?
@@ -212,6 +216,8 @@ function tear_down_database() {
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
 function tear_down() {
+    reset_default_database
+
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
     sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');"
@@ -362,7 +368,7 @@ function test_cdb_tablemetadatatouch_fails_from_user_without_permission() {
 
 function test_cdb_tablemetadatatouch_fully_qualifies_names() {
     sql postgres "CREATE TABLE touch_invalidations (table_name text);"
-    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
+    sql postgres "create or replace function cartodb.cdb_invalidate_varnish(table_name text) returns void as \$\$ begin insert into public.touch_invalidations select table_name; end; \$\$ language 'plpgsql';"
 
     #default schema
     sql "CREATE TABLE touch_example (a int);"
@@ -532,6 +538,8 @@ END
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);"
     DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);"
 
+    reset_default_database
+
     # Add PGPORT to conf if it is set
     PORT_SPEC=""
     if [[ "$PGPORT" != "" ]] ; then
@@ -659,8 +667,10 @@ EOF
     DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;'
     DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;'
 
+    reset_default_database
     sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';"
     DATABASE=fdw_target tear_down_database
+    reset_default_database
 }
 
 function test_cdb_catalog_basic_node() {

util/create_from_unpackaged.sh

@@ -10,6 +10,7 @@ echo "-- Script generated by $0 on `date`" > ${output}
 cat ${input} |
   grep '^ *CREATE OR REPLACE FUNCTION' |
   grep -v ' cartodb\.' | # should  only match DDL hooks
+  grep -v '.*\quit.*' | 
   sed 's/).*$/)/' |
   sed 's/DEFAULT [^ ,)]*//g' |
   sed 's/CREATE OR REPLACE FUNCTION /ALTER FUNCTION public./' |
@@ -19,59 +20,4 @@ cat ${input} |
   cat >> ${output}
 
 # Upgrade all functions
-cat ${input} | grep -v 'duplicated extension$' >> ${output}
-
-# Migrate CDB_TableMetadata
-cat >> ${output} <<'EOF'
-ALTER TABLE cartodb.CDB_TableMetadata DISABLE TRIGGER ALL;
-INSERT INTO cartodb.CDB_TableMetadata SELECT * FROM public.CDB_TableMetadata;
-ALTER TABLE cartodb.CDB_TableMetadata ENABLE TRIGGER ALL;
-DROP TABLE public.CDB_TableMetadata;
-
--- Set user quota
--- NOTE: will fail if user quota wasn't set at database level, see
---       http://github.com/CartoDB/cartodb-postgresql/issues/18
-DO $$
-DECLARE
-  qmax int8;
-BEGIN
-  BEGIN
-    qmax := public._CDB_UserQuotaInBytes();
-  EXCEPTION WHEN undefined_function THEN
-    RAISE EXCEPTION 'Please set user quota before switching to cartodb extension';
-  END;
-  PERFORM cartodb.CDB_SetUserQuotaInBytes(qmax);
-  DROP FUNCTION public._CDB_UserQuotaInBytes();
-END;
-$$ LANGUAGE 'plpgsql';
-EOF
-
-## Cartodbfy tables with a trigger using 'CDB_CheckQuota' or
-## 'CDB_TableMetadata_Trigger' from the 'public' schema
-#cat >> ${output} <<'EOF'
-#select cartodb.CDB_CartodbfyTable(relname::regclass) from ( 
-#  -- names of tables using public.CDB_CheckQuota or
-#  -- public.CDB_TableMetadata_Trigger in their triggers
-#  SELECT distinct c.relname
-#  FROM
-#    pg_trigger t,
-#    pg_class c,
-#    pg_proc p,
-#    pg_namespace n
-#  WHERE
-#    n.nspname = 'public' AND
-#    p.pronamespace = n.oid AND
-#    p.proname IN ( 'cdb_checkquota', 'cdb_tablemetadata_trigger' ) AND
-#    t.tgrelid = c.oid AND
-#    p.oid = t.tgfoid 
-#) as foo;
-#EOF
-
-## Drop any leftover function from public schema (there should be none)
-#cat ${input} |
-#  grep '^ *CREATE OR REPLACE FUNCTION' |
-#  grep -v ' cartodb\.' | # should  only match DDL hooks
-#  sed 's/).*$/);/' |
-#  sed 's/DEFAULT [^ ,)]*//g' |
-#  sed 's/CREATE OR REPLACE FUNCTION /DROP FUNCTION IF EXISTS public./' |
-#  cat >> ${output}
+cat ${input} | grep -v 'duplicated extension$' | grep -v '\quit$' | grep -v 'pg_extension_config_dump' >> ${output}