From 48025013872fd87ec8fbc4aa62b7cc88719cf701 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 11:59:42 +0200 Subject: [PATCH 01/25] Travis: Test against PG12 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f448081..396d111 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - POSTGRESQL_VERSION="9.6" - POSTGRESQL_VERSION="10" - POSTGRESQL_VERSION="11" + - POSTGRESQL_VERSION="12" before_install: From 9f904b59264105604721e36179cc365275818dc2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 12:12:45 +0200 Subject: [PATCH 02/25] plpython2 is dead, long live plpython --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 396d111..a115286 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,23 +17,26 @@ env: - POSTGRESQL_VERSION="12" -before_install: +script: - sudo service postgresql stop; - sudo apt-get remove postgresql* -y - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION + - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis + # For pre12, install plpython2. For PG12 install plpython3 and make it default + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo apt-get install -y postgresql-plpython3-12 && sudo cp /usr/share/postgresql/12/extension/plpython3u.control /usr/share/postgresql/12/extension/plpythonu.control && sudo cp /usr/share/postgresql/12/extension/plpython3u--1.0.sql /usr/share/postgresql/12/extension/plpythonu--1.0.sql && sudo cp /usr/share/postgresql/12/extension/plpython3u--unpackaged--1.0.sql /usr/share/postgresql/12/extension/plpythonu--unpackaged--1.0.sql; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - sudo pip install redis==2.4.9 -script: + - make - sudo make install - make installcheck after_failure: + - dpkg-query -L postgresql-plpython3-$POSTGRESQL_VERSION - pg_lsclusters - cat regression.out - cat regression.diffs From 3b4669df1150b84df79292432a5701ae855fa642 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 14:24:20 +0200 Subject: [PATCH 03/25] Try hacking plpythonu -> plpython3u --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a115286..e2358a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,19 +24,16 @@ script: - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 and make it default - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo apt-get install -y postgresql-plpython3-12 && sudo cp /usr/share/postgresql/12/extension/plpython3u.control /usr/share/postgresql/12/extension/plpythonu.control && sudo cp /usr/share/postgresql/12/extension/plpython3u--1.0.sql /usr/share/postgresql/12/extension/plpythonu--1.0.sql && sudo cp /usr/share/postgresql/12/extension/plpython3u--unpackaged--1.0.sql /usr/share/postgresql/12/extension/plpythonu--unpackaged--1.0.sql; fi; + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION && sudo pip install redis==2.4.9; else sudo apt-get install -y postgresql-plpython3-12 && find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/plpythonu/plpython3u/g' && pip3 install redis; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - - sudo pip install redis==2.4.9 - - make - sudo make install - make installcheck after_failure: - - dpkg-query -L postgresql-plpython3-$POSTGRESQL_VERSION - pg_lsclusters - cat regression.out - cat regression.diffs From aed88fd958a7e38f07e820828ba97f208cdf31d2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 15:29:45 +0200 Subject: [PATCH 04/25] Force all PG12 to use plpython3u --- .travis.yml | 2 +- Makefile | 12 +++-- NEWS.md | 3 ++ test/CDB_OAuth_expect.pg12 | 90 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 test/CDB_OAuth_expect.pg12 diff --git a/.travis.yml b/.travis.yml index e2358a3..58847e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ script: - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 and make it default - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION && sudo pip install redis==2.4.9; else sudo apt-get install -y postgresql-plpython3-12 && find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/plpythonu/plpython3u/g' && pip3 install redis; fi; + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION && sudo pip install redis==2.4.9; else sudo apt-get install -y postgresql-plpython3-12 && pip3 install redis==2.4.9; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password diff --git a/Makefile b/Makefile index f59fa62..885e0da 100644 --- a/Makefile +++ b/Makefile @@ -134,12 +134,18 @@ PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := "plpythonu" +ifeq ($(PG_12_GE), true) +PLPYTHONU := "plpython3u" +endif $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@ cat $(CDBSCRIPTS) | \ $(SED) -e 's/@extschema@/cartodb/g' \ - -e "s/@postgisschema@/public/g" >> $@ + -e 's/@postgisschema@/public/g' \ + -e 's/plpythonu/$(PLPYTHONU)/g' >> $@ echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@ cat cartodb_version.sql >> $@ @@ -153,10 +159,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION). cp $< $@ $(EXTENSION).control: $(EXTENSION).control.in Makefile - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ # Needed for consistent `echo` results with backslashes SHELL = bash diff --git a/NEWS.md b/NEWS.md index 0ebb16e..3213def 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +0.32.0 (XXXX-XX-XX) +* Make PG12 depend on plpython3u instead of plpythonu + 0.31.0 (2019-10-08) * Ghost tables: Add missing tags (#370) * Set search_path in security definer functions. diff --git a/test/CDB_OAuth_expect.pg12 b/test/CDB_OAuth_expect.pg12 new file mode 100644 index 0000000..73767d2 --- /dev/null +++ b/test/CDB_OAuth_expect.pg12 @@ -0,0 +1,90 @@ + +CREATE TABLE +INSERT 0 1 +SELECT 1 +CREATE VIEW +SELECT 1 +SELECT 1 +1 +1 +1 +1 +1 +ERROR: permission denied for table test +ERROR: permission denied for table test_tablesas +ERROR: permission denied for view test_view +ERROR: permission denied for materialized view test_mview +ERROR: permission denied for table test_selectinto +DROP TABLE +DROP VIEW +DROP MATERIALIZED VIEW +DROP TABLE +DROP TABLE +NOTICE: event trigger "oauth_reassign_tables_trigger" does not exist, skipping + +CREATE TABLE +INSERT 0 1 +SELECT 1 +CREATE VIEW +SELECT 1 +SELECT 1 +1 +1 +1 +1 +1 +ERROR: permission denied for table test2 +ERROR: permission denied for table test2_tablesas +ERROR: permission denied for view test2_view +ERROR: permission denied for materialized view test2_mview +ERROR: permission denied for table test2_selectinto +DROP TABLE +DROP VIEW +DROP MATERIALIZED VIEW +DROP TABLE +DROP TABLE + +CREATE TABLE +INSERT 0 1 +SELECT 1 +CREATE VIEW +SELECT 1 +SELECT 1 +1 +1 +1 +1 +1 +ERROR: permission denied for table test3 +ERROR: permission denied for table test3_tablesas +ERROR: permission denied for view test3_view +ERROR: permission denied for materialized view test3_mview +ERROR: permission denied for table test3_selectinto +DROP TABLE +DROP VIEW +DROP MATERIALIZED VIEW +DROP TABLE +DROP TABLE + +CREATE TABLE +INSERT 0 1 +SELECT 1 +CREATE VIEW +SELECT 1 +SELECT 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +DROP TABLE +DROP VIEW +DROP MATERIALIZED VIEW +DROP TABLE +DROP TABLE + From ccbaf785cf9272ba0f73937b99ba3be8976a4844 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 15:49:31 +0200 Subject: [PATCH 05/25] Updates --- .travis.yml | 2 +- Makefile | 4 ++-- README.md | 2 +- expected/test_setup.out | 4 +--- sql/test_setup.sql | 4 +--- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58847e0..56bbbf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ script: - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis - # For pre12, install plpython2. For PG12 install plpython3 and make it default + # For pre12, install plpython2. For PG12 install plpython3 - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION && sudo pip install redis==2.4.9; else sudo apt-get install -y postgresql-plpython3-12 && pip3 install redis==2.4.9; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION diff --git a/Makefile b/Makefile index 885e0da..6660b46 100644 --- a/Makefile +++ b/Makefile @@ -135,9 +135,9 @@ include $(PGXS) PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) -PLPYTHONU := "plpythonu" +PLPYTHONU := plpythonu ifeq ($(PG_12_GE), true) -PLPYTHONU := "plpython3u" +PLPYTHONU := plpython3u endif $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile diff --git a/README.md b/README.md index a871807..b4a7b8b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/ Dependencies ------------ - * PostgreSQL 9.6+ (with plpythonu extension and xml support) + * PostgreSQL 9.6+ (with plpythonu extension and xml support). For PostgreSQL 12+ plpython3u is required instead of plpythonu. * [PostGIS extension](http://postgis.net) * Python with [Redis module](https://pypi.org/project/redis/) diff --git a/expected/test_setup.out b/expected/test_setup.out index cb4d895..aa08fe7 100644 --- a/expected/test_setup.out +++ b/expected/test_setup.out @@ -1,6 +1,4 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; +CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ BEGIN diff --git a/sql/test_setup.sql b/sql/test_setup.sql index cb4d895..aa08fe7 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -1,6 +1,4 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; +CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ BEGIN From 0cecfdb112d3c0ed50782f9fa8b6dacba30d7454 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 16:16:16 +0200 Subject: [PATCH 06/25] log level --- expected/test_setup.out | 1 + sql/test_setup.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/expected/test_setup.out b/expected/test_setup.out index aa08fe7..6cea6c0 100644 --- a/expected/test_setup.out +++ b/expected/test_setup.out @@ -1,3 +1,4 @@ +SET client_min_messages TO error; CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ diff --git a/sql/test_setup.sql b/sql/test_setup.sql index aa08fe7..6cea6c0 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -1,3 +1,4 @@ +SET client_min_messages TO error; CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ From 9686ad5cfeaee7e2b9abc54645572e631f1c0ea7 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 16:49:08 +0200 Subject: [PATCH 07/25] Try to work with both python2 and python3 --- scripts-available/CDB_Groups_API.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index eda868a..5050d86 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -189,7 +189,10 @@ CREATE OR REPLACE FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[]) RETURNS int AS $$ - import httplib + try: + import httplib as client + except: + from http import client params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0] if params['host'] is None: @@ -202,17 +205,17 @@ $$ last_err = None while retry > 0: try: - client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout']) + conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout']) database_name = plpy.execute("select current_database();")[0]['current_database'] - client.request(method, url.format(database_name), body, headers) - response = client.getresponse() + conn.request(method, url.format(database_name), body, headers) + response = conn.getresponse() assert response.status in valid_return_codes return response.status except Exception as err: retry -= 1 last_err = err plpy.warning('Retrying after: ' + str(err)) - client = SD['groups_api_client'] = None + conn = SD['groups_api_client'] = None if last_err is not None: plpy.error('Fatal Group API error: ' + str(last_err)) From 8fac0c89eb2aca8c95b1ef2cfd8f354d52470706 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 17:48:46 +0200 Subject: [PATCH 08/25] python3: Try to adapt pathname2url --- scripts-available/CDB_Groups_API.sql | 42 ++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index 5050d86..660d4e8 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -33,9 +33,12 @@ FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text) RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(group_name)) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url plpy.execute(query) @@ -50,9 +53,12 @@ FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_n RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name)) + url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(old_group_name)) body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role) query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -67,9 +73,12 @@ FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[]) RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name)) body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -84,9 +93,12 @@ FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[ RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name)) body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body) plpy.execute(query) @@ -109,9 +121,12 @@ FUNCTION @extschema@._CDB_Group_Table_GrantPermission_API(group_name text, usern RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name) + url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name) body = '{ "access": "%s" }' % access query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -134,9 +149,12 @@ FUNCTION @extschema@._CDB_Group_Table_RevokeAllPermission_API(group_name text, u RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name) + url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url plpy.execute(query) $$ LANGUAGE 'plpythonu' From 1a99151d0e32b0ff0c8369c2b5c0dd571a4270bb Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 18:23:27 +0200 Subject: [PATCH 09/25] Install pip and pip3 --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56bbbf1..c7254c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ env: - POSTGRESQL_VERSION="11" - POSTGRESQL_VERSION="12" - script: - sudo service postgresql stop; - sudo apt-get remove postgresql* -y @@ -24,7 +23,10 @@ script: - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION && sudo pip install redis==2.4.9; else sudo apt-get install -y postgresql-plpython3-12 && pip3 install redis==2.4.9; fi; + - sudo apt-get install python-pip python3-pip + - sudo pip install redis==2.4.9; + - sudo pip3 install redis==2.4.9; + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo apt-get install -y postgresql-plpython3-12; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password From cb019ff194c0a7dc7841cd1c317630cf092d2c64 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 18:35:51 +0200 Subject: [PATCH 10/25] Install python3 for PG12 run --- .travis.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7254c6..90a9221 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,12 @@ env: - PGPORT=5432 - POSTGIS_VERSION="2.5" - matrix: - - POSTGRESQL_VERSION="9.6" - - POSTGRESQL_VERSION="10" - - POSTGRESQL_VERSION="11" - - POSTGRESQL_VERSION="12" +jobs: + include: + - env: POSTGRESQL_VERSION="9.6" + - env: POSTGRESQL_VERSION="10" + - env: POSTGRESQL_VERSION="11" + - env: POSTGRESQL_VERSION="12" script: - sudo service postgresql stop; @@ -23,10 +24,8 @@ script: - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 - - sudo apt-get install python-pip python3-pip - - sudo pip install redis==2.4.9; - - sudo pip3 install redis==2.4.9; - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo apt-get install -y postgresql-plpython3-12; fi; + - sudo apt-get install -y python python-pip python3 python3-pip + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo pip install redis; sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo pip3 install pip3; sudo pip3 install setuptools; sudo pip3 install redis; sudo apt-get install -y postgresql-plpython3-12; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password From d1eb740a00391d9fdd1110fd3d2764142349d3e5 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 19:45:42 +0200 Subject: [PATCH 11/25] Missing qualifications --- scripts-available/CDB_Overviews.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index ec4d307..d45a5a3 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -662,7 +662,7 @@ AS $$ offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m); END IF; - point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y); + point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y); -- compute the resulting columns in the same order as in the base table WITH cols AS ( From 9248ce76bdcb66b661c20b1c91eb7c59555ac936 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 19:48:00 +0200 Subject: [PATCH 12/25] Update pip3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 90a9221..749f8b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ script: - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 - sudo apt-get install -y python python-pip python3 python3-pip - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo pip install redis; sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo pip3 install pip3; sudo pip3 install setuptools; sudo pip3 install redis; sudo apt-get install -y postgresql-plpython3-12; fi; + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo pip install redis; sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo pip3 install pip; sudo pip3 install setuptools; sudo pip3 install redis; sudo apt-get install -y postgresql-plpython3-12; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password From 82451560380c7cdc61eb5fd74dab43540934bd49 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 20:06:20 +0200 Subject: [PATCH 13/25] Use distribution package for python redis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 749f8b3..94cc758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,7 @@ script: - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis # For pre12, install plpython2. For PG12 install plpython3 - - sudo apt-get install -y python python-pip python3 python3-pip - - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo pip install redis; sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION; else sudo pip3 install pip; sudo pip3 install setuptools; sudo pip3 install redis; sudo apt-get install -y postgresql-plpython3-12; fi; + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password From e787b1d097ad50f523c32344d2c39b3e51082a31 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 20:09:15 +0200 Subject: [PATCH 14/25] Missing qualification --- scripts-available/CDB_Quota.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_Quota.sql b/scripts-available/CDB_Quota.sql index d3dc983..d11f400 100644 --- a/scripts-available/CDB_Quota.sql +++ b/scripts-available/CDB_Quota.sql @@ -21,8 +21,9 @@ $$ DECLARE total_size INT8; BEGIN + -- TODO: Make this compatible with postgis without raster (No view and no raster tables) WITH raster_tables AS ( - SELECT o_table_name, r_table_name FROM raster_overviews + SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews WHERE o_table_schema = schema_name AND o_table_catalog = current_database() ), user_tables AS ( From a4bcf4c78f73cfb304e6a9750961d0d087fe9526 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 22 Oct 2019 20:13:38 +0200 Subject: [PATCH 15/25] Update error messages --- test/CDB_GhostTables_expect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 01156f7..469edeb 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -4,7 +4,7 @@ WARNING: Invalidation service configuration not found. Skipping Ghost Tables li NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known. +WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER BEGIN @@ -12,7 +12,7 @@ cdb_ddl_execution 0 CREATE TABLE 1 -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known. +WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE COMMIT From 34684507cbf94497202fc60ee0729952290e430e Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 13:41:38 +0200 Subject: [PATCH 16/25] Cast sql_identifier to text --- scripts-available/CDB_Overviews.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index d45a5a3..c73d873 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -397,7 +397,7 @@ DECLARE attr_list TEXT; BEGIN SELECT string_agg(s.c, ',') FROM ( - SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c + SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c ) AS s INTO attr_list; RETURN attr_list; @@ -554,7 +554,7 @@ DECLARE BEGIN SELECT string_agg(@extschema@._CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',') FROM ( - SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c + SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c ) AS s INTO attr_list; RETURN attr_list; @@ -673,7 +673,7 @@ AS $$ Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom) WHEN 'the_geom_webmercator' THEN Format('%s AS the_geom_webmercator', point_geom) - ELSE c + ELSE c::text END AS column FROM @extschema@.CDB_ColumnNames(reloid) c ) @@ -800,7 +800,7 @@ AS $$ '@postgisschema@.ST_Transform(@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom' WHEN 'the_geom_webmercator' THEN '@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator' - ELSE c + ELSE c::text END AS column FROM CDB_ColumnNames(reloid) c ) @@ -924,7 +924,7 @@ AS $$ SELECT CASE c WHEN 'cartodb_id' THEN 'cartodb_id' - ELSE c + ELSE c::text END AS column FROM @extschema@.CDB_ColumnNames(reloid) c ) From 29397ff9293b588533d18927e629a0e9b08d2079 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 14:21:45 +0200 Subject: [PATCH 17/25] Make it work without postgis_raster --- scripts-available/CDB_Quota.sql | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_Quota.sql b/scripts-available/CDB_Quota.sql index d11f400..2224891 100644 --- a/scripts-available/CDB_Quota.sql +++ b/scripts-available/CDB_Quota.sql @@ -15,39 +15,55 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; -- Return the estimated size of user data. Used for quota checking. -CREATE OR REPLACE FUNCTION @extschema@.CDB_UserDataSize(schema_name TEXT) +CREATE OR REPLACE FUNCTION cartodb.CDB_UserDataSize(schema_name TEXT) RETURNS bigint AS $$ DECLARE total_size INT8; + raster_available BOOLEAN; + raster_read_query TEXT; BEGIN - -- TODO: Make this compatible with postgis without raster (No view and no raster tables) + -- Postgis 3+ might not install raster + raster_available := EXISTS ( + SELECT 1 + FROM pg_views + WHERE schemaname = '@postgisschema@' + AND viewname = 'raster_overviews' + ); + + IF raster_available THEN + raster_read_query := Format('SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews + WHERE o_table_schema = ''%I'' AND o_table_catalog = current_database()', schema_name); + ELSE + raster_read_query := 'SELECT NULL::text AS o_table_name, NULL::text AS r_table_name'; + END IF; + EXECUTE Format(' WITH raster_tables AS ( - SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews - WHERE o_table_schema = schema_name AND o_table_catalog = current_database() + %s ), user_tables AS ( - SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(schema_name) + SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(''%I'') ), table_cat AS ( SELECT table_name, ( EXISTS(select * from raster_tables where o_table_name = table_name) - OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || '[\w\d]*' + OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || ''[\w\d]*'' ) AS is_overview, EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster FROM user_tables ), sizes AS ( - SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(schema_name, table_name)))) table_size, + SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(''%I'', table_name)))) table_size, CASE WHEN is_overview THEN 0 WHEN is_raster THEN 1 ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator END AS multiplier FROM table_cat GROUP BY is_overview, is_raster ) - SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes; + SELECT sum(table_size*multiplier)::int8 FROM sizes + ', raster_read_query, schema_name, schema_name) INTO total_size; IF total_size IS NOT NULL THEN RETURN total_size; From 021c7318a3a8851a7a7fc9c874e4df656ca45e0a Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 16:49:57 +0200 Subject: [PATCH 18/25] Adapt QuotaTest to avoid randoms --- test/CDB_QuotaTest.sql | 27 +++++++++++++++------------ test/CDB_QuotaTest_expect | 5 +++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test/CDB_QuotaTest.sql b/test/CDB_QuotaTest.sql index 92b65d1..61553ad 100644 --- a/test/CDB_QuotaTest.sql +++ b/test/CDB_QuotaTest.sql @@ -1,30 +1,33 @@ set client_min_messages to error; \set VERBOSITY TERSE --- See the dice -SELECT setseed(0.5); - CREATE TABLE big(a int); -- Try the legacy interface -- See https://github.com/CartoDB/cartodb-postgresql/issues/13 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big - EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public'); + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); INSERT INTO big VALUES (1); -- allowed, check runs before INSERT INTO big VALUES (2); -- disallowed, quota exceeds before -SELECT CDB_SetUserQuotaInBytes(0); -SELECT CDB_CartodbfyTable('big'); +SELECT cartodb.CDB_SetUserQuotaInBytes(0); +SELECT cartodb.CDB_CartodbfyTable('big'); +-- Creating the trigger should fail as it was created by CDB_CartodbfyTable +CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); +-- Drop the trigger and recreate it forcing a 100% checks +DROP TRIGGER test_quota ON big; +CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); INSERT INTO big SELECT generate_series(2049,4096); INSERT INTO big SELECT generate_series(4097,6144); INSERT INTO big SELECT generate_series(6145,8192); -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108 -SELECT CDB_UserDataSize(); +SELECT cartodb.CDB_UserDataSize(); SELECT cartodb._CDB_total_relation_size('public', 'big'); SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name'); -- END Test for #108 -SELECT setseed(0.9); -SELECT CDB_SetUserQuotaInBytes(2); +SELECT cartodb.CDB_SetUserQuotaInBytes(2); INSERT INTO big VALUES (8193); -SELECT CDB_SetUserQuotaInBytes(0); +SELECT cartodb.CDB_SetUserQuotaInBytes(0); INSERT INTO big VALUES (8194); DROP TABLE big; @@ -32,8 +35,8 @@ DROP TABLE big; --analysis tables should be excluded from quota: CREATE TABLE big(a int); CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big - EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public'); -SELECT CDB_SetUserQuotaInBytes(1); + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); +SELECT cartodb.CDB_SetUserQuotaInBytes(1); CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int); INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5); INSERT INTO big VALUES (1); -- allowed, check runs before diff --git a/test/CDB_QuotaTest_expect b/test/CDB_QuotaTest_expect index 9204514..2cc56dc 100644 --- a/test/CDB_QuotaTest_expect +++ b/test/CDB_QuotaTest_expect @@ -1,18 +1,19 @@ SET - CREATE TABLE CREATE TRIGGER INSERT 0 1 ERROR: Quota exceeded by 3.9990234375KB 0 big +ERROR: trigger "test_quota" for relation "big" already exists +DROP TRIGGER +CREATE TRIGGER INSERT 0 2048 INSERT 0 2048 INSERT 0 2048 454656 909312 0 - 2 ERROR: Quota exceeded by 443.998046875KB 0 From 946987102909d650c13b17e77b8f24671219cd3a Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 17:06:47 +0200 Subject: [PATCH 19/25] Remove unneeded change --- scripts-available/CDB_Quota.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_Quota.sql b/scripts-available/CDB_Quota.sql index 2224891..96b41f2 100644 --- a/scripts-available/CDB_Quota.sql +++ b/scripts-available/CDB_Quota.sql @@ -15,7 +15,7 @@ $$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE; -- Return the estimated size of user data. Used for quota checking. -CREATE OR REPLACE FUNCTION cartodb.CDB_UserDataSize(schema_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.CDB_UserDataSize(schema_name TEXT) RETURNS bigint AS $$ DECLARE From dac1f5bffcf7083f71edaf082ef25b986f10b487 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 17:15:50 +0200 Subject: [PATCH 20/25] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 3213def..2ccee9e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ 0.32.0 (XXXX-XX-XX) +* Some fixes for PG12. * Make PG12 depend on plpython3u instead of plpythonu +* CDB_UserDataSize is now compatible with postgis 3 without postgis_raster. 0.31.0 (2019-10-08) * Ghost tables: Add missing tags (#370) From 21c319eb9a1c70d2c4e83d9eebf94cb5bf6be7e6 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 23 Oct 2019 17:18:50 +0200 Subject: [PATCH 21/25] Add postgis 3 build --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94cc758..729d84f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,14 +8,15 @@ env: - PGDATABASE=postgres - PGOPTIONS='-c client_min_messages=NOTICE' - PGPORT=5432 - - POSTGIS_VERSION="2.5" jobs: include: - - env: POSTGRESQL_VERSION="9.6" - - env: POSTGRESQL_VERSION="10" - - env: POSTGRESQL_VERSION="11" - - env: POSTGRESQL_VERSION="12" + - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" + script: - sudo service postgresql stop; From 409b0677213d2351c78f9e346553706787a144d6 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 24 Oct 2019 11:15:15 +0200 Subject: [PATCH 22/25] Quota: Remove kb error from regress tests --- test/CDB_QuotaTest.sql | 26 +++++++++++++++++++++----- test/CDB_QuotaTest_expect | 12 +++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test/CDB_QuotaTest.sql b/test/CDB_QuotaTest.sql index 61553ad..bbb8753 100644 --- a/test/CDB_QuotaTest.sql +++ b/test/CDB_QuotaTest.sql @@ -1,5 +1,19 @@ set client_min_messages to error; \set VERBOSITY TERSE +-- Runs a query and returns whether an error was thrown +-- Useful when the error message depends on the execution plan or db settings +-- The error message outputs the extra quota, and this might depend on the database setup and version +CREATE OR REPLACE FUNCTION catch_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN OTHERS THEN + RETURN TRUE; +END +$$ LANGUAGE 'plpgsql'; CREATE TABLE big(a int); -- Try the legacy interface @@ -7,7 +21,7 @@ CREATE TABLE big(a int); CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); INSERT INTO big VALUES (1); -- allowed, check runs before -INSERT INTO big VALUES (2); -- disallowed, quota exceeds before +SELECT 'excess1', catch_error($$INSERT INTO big VALUES (2); $$); -- disallowed, quota exceeds before SELECT cartodb.CDB_SetUserQuotaInBytes(0); SELECT cartodb.CDB_CartodbfyTable('big'); -- Creating the trigger should fail as it was created by CDB_CartodbfyTable @@ -21,12 +35,13 @@ INSERT INTO big SELECT generate_series(2049,4096); INSERT INTO big SELECT generate_series(4097,6144); INSERT INTO big SELECT generate_series(6145,8192); -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108 -SELECT cartodb.CDB_UserDataSize(); -SELECT cartodb._CDB_total_relation_size('public', 'big'); +SELECT cartodb.CDB_UserDataSize() < 500000 AND cartodb.CDB_UserDataSize() > 0; +SELECT cartodb._CDB_total_relation_size('public', 'big') < 1000000; SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name'); -- END Test for #108 + SELECT cartodb.CDB_SetUserQuotaInBytes(2); -INSERT INTO big VALUES (8193); +SELECT 'excess2', catch_error($$INSERT INTO big VALUES (8193);$$); SELECT cartodb.CDB_SetUserQuotaInBytes(0); INSERT INTO big VALUES (8194); DROP TABLE big; @@ -41,10 +56,11 @@ CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5); INSERT INTO big VALUES (1); -- allowed, check runs before DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4; -INSERT INTO big VALUES (2); -- disallowed, quota exceeds before +SELECT 'excess3', catch_error($$INSERT INTO big VALUES (3);$$); -- disallowed, quota exceeds before DROP TABLE big; SELECT CDB_SetUserQuotaInBytes(0); set client_min_messages to NOTICE; +DROP FUNCTION catch_error(text); DROP FUNCTION _CDB_UserQuotaInBytes(); diff --git a/test/CDB_QuotaTest_expect b/test/CDB_QuotaTest_expect index 2cc56dc..49f9e7d 100644 --- a/test/CDB_QuotaTest_expect +++ b/test/CDB_QuotaTest_expect @@ -1,8 +1,9 @@ SET +CREATE FUNCTION CREATE TABLE CREATE TRIGGER INSERT 0 1 -ERROR: Quota exceeded by 3.9990234375KB +excess1|t 0 big ERROR: trigger "test_quota" for relation "big" already exists @@ -11,11 +12,11 @@ CREATE TRIGGER INSERT 0 2048 INSERT 0 2048 INSERT 0 2048 -454656 -909312 +t +t 0 2 -ERROR: Quota exceeded by 443.998046875KB +excess2|t 0 INSERT 0 1 DROP TABLE @@ -26,8 +27,9 @@ CREATE TABLE INSERT 0 5 INSERT 0 1 DROP TABLE -ERROR: Quota exceeded by 3.9990234375KB +excess3|t DROP TABLE 0 SET DROP FUNCTION +DROP FUNCTION From 7f9a74811950d1849e3d8a5e015b2333aaaa2ea0 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 24 Oct 2019 11:33:07 +0200 Subject: [PATCH 23/25] Make CDB_OAuth independent of the PG version --- Makefile | 6 +-- test/CDB_OAuth.sql | 48 +++++++++++++------- test/CDB_OAuth_expect | 30 ++++++------- test/CDB_OAuth_expect.pg11 | 90 -------------------------------------- test/CDB_OAuth_expect.pg12 | 90 -------------------------------------- 5 files changed, 49 insertions(+), 215 deletions(-) delete mode 100644 test/CDB_OAuth_expect.pg11 delete mode 100644 test/CDB_OAuth_expect.pg12 diff --git a/Makefile b/Makefile index 6660b46..db588aa 100644 --- a/Makefile +++ b/Makefile @@ -182,11 +182,7 @@ legacy_regress: $(REGRESS_OLD) Makefile $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \ exp=expected/test/$${tn}.out; \ echo '\set ECHO none' > $${exp}; \ - if [[ -f "test/$${tn}_expect.pg$(PG_VERSION)" ]]; then \ - cat test/$${tn}_expect.pg$(PG_VERSION) >> $${exp}; \ - else \ - cat test/$${tn}_expect >> $${exp}; \ - fi \ + cat test/$${tn}_expect >> $${exp}; \ done test_organization: diff --git a/test/CDB_OAuth.sql b/test/CDB_OAuth.sql index c3c14e6..3a32b3b 100644 --- a/test/CDB_OAuth.sql +++ b/test/CDB_OAuth.sql @@ -1,6 +1,23 @@ -- Create user and enable OAuth event trigger \set QUIET on SET client_min_messages TO error; + +-- The permission error changed between pre PG11 and post 11 (before everythin "relation", now it's "view", "table" and so on +CREATE OR REPLACE FUNCTION catch_permission_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN insufficient_privilege THEN + RETURN TRUE; + WHEN OTHERS THEN + RAISE WARNING 'Exception %', sqlstate; + RETURN FALSE; +END +$$ LANGUAGE 'plpgsql'; + DROP ROLE IF EXISTS "creator_role"; CREATE ROLE "creator_role" LOGIN; DROP ROLE IF EXISTS "ownership_role"; @@ -30,11 +47,11 @@ SELECT * FROM test_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test; -SELECT * FROM test_tablesas; -SELECT * FROM test_view; -SELECT * FROM test_mview; -SELECT * FROM test_selectinto; +SELECT 'denied_table', catch_permission_error($$SELECT * FROM test;$$); +SELECT 'denied_tableas', catch_permission_error($$SELECT * FROM test_tablesas;$$); +SELECT 'denied_view', catch_permission_error($$SELECT * FROM test_view;$$); +SELECT 'denied_mview', catch_permission_error($$SELECT * FROM test_mview;$$); +SELECT 'denied_selectinto', catch_permission_error($$SELECT * FROM test_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -71,11 +88,11 @@ SELECT * FROM test2_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test2; -SELECT * FROM test2_tablesas; -SELECT * FROM test2_view; -SELECT * FROM test2_mview; -SELECT * FROM test2_selectinto; +SELECT 'denied_table2', catch_permission_error($$SELECT * FROM test2;$$); +SELECT 'denied_tableas2', catch_permission_error($$SELECT * FROM test2_tablesas;$$); +SELECT 'denied_view2', catch_permission_error($$SELECT * FROM test2_view;$$); +SELECT 'denied_mview2', catch_permission_error($$SELECT * FROM test2_mview;$$); +SELECT 'denied_selectinto2', catch_permission_error($$SELECT * FROM test2_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -112,11 +129,11 @@ SELECT * FROM test3_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test3; -SELECT * FROM test3_tablesas; -SELECT * FROM test3_view; -SELECT * FROM test3_mview; -SELECT * FROM test3_selectinto; +SELECT 'denied_table3', catch_permission_error($$SELECT * FROM test3;$$); +SELECT 'denied_tableas3', catch_permission_error($$SELECT * FROM test3_tablesas;$$); +SELECT 'denied_view3', catch_permission_error($$SELECT * FROM test3_view;$$); +SELECT 'denied_mview3', catch_permission_error($$SELECT * FROM test3_mview;$$); +SELECT 'denied_selectinto3', catch_permission_error($$SELECT * FROM test3_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -174,4 +191,5 @@ DROP ROLE "ownership_role"; REVOKE ALL ON SCHEMA cartodb FROM "creator_role"; DROP ROLE "creator_role"; DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role'; +DROP FUNCTION catch_permission_error(text); \set QUIET off diff --git a/test/CDB_OAuth_expect b/test/CDB_OAuth_expect index f536c43..1a334c0 100644 --- a/test/CDB_OAuth_expect +++ b/test/CDB_OAuth_expect @@ -10,11 +10,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test -ERROR: permission denied for relation test_tablesas -ERROR: permission denied for relation test_view -ERROR: permission denied for relation test_mview -ERROR: permission denied for relation test_selectinto +denied_table|t +denied_tableas|t +denied_view|t +denied_mview|t +denied_selectinto|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW @@ -33,11 +33,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test2 -ERROR: permission denied for relation test2_tablesas -ERROR: permission denied for relation test2_view -ERROR: permission denied for relation test2_mview -ERROR: permission denied for relation test2_selectinto +denied_table2|t +denied_tableas2|t +denied_view2|t +denied_mview2|t +denied_selectinto2|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW @@ -55,11 +55,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test3 -ERROR: permission denied for relation test3_tablesas -ERROR: permission denied for relation test3_view -ERROR: permission denied for relation test3_mview -ERROR: permission denied for relation test3_selectinto +denied_table3|t +denied_tableas3|t +denied_view3|t +denied_mview3|t +denied_selectinto3|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW diff --git a/test/CDB_OAuth_expect.pg11 b/test/CDB_OAuth_expect.pg11 deleted file mode 100644 index 73767d2..0000000 --- a/test/CDB_OAuth_expect.pg11 +++ /dev/null @@ -1,90 +0,0 @@ - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test -ERROR: permission denied for table test_tablesas -ERROR: permission denied for view test_view -ERROR: permission denied for materialized view test_mview -ERROR: permission denied for table test_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE -NOTICE: event trigger "oauth_reassign_tables_trigger" does not exist, skipping - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test2 -ERROR: permission denied for table test2_tablesas -ERROR: permission denied for view test2_view -ERROR: permission denied for materialized view test2_mview -ERROR: permission denied for table test2_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test3 -ERROR: permission denied for table test3_tablesas -ERROR: permission denied for view test3_view -ERROR: permission denied for materialized view test3_mview -ERROR: permission denied for table test3_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - diff --git a/test/CDB_OAuth_expect.pg12 b/test/CDB_OAuth_expect.pg12 deleted file mode 100644 index 73767d2..0000000 --- a/test/CDB_OAuth_expect.pg12 +++ /dev/null @@ -1,90 +0,0 @@ - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test -ERROR: permission denied for table test_tablesas -ERROR: permission denied for view test_view -ERROR: permission denied for materialized view test_mview -ERROR: permission denied for table test_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE -NOTICE: event trigger "oauth_reassign_tables_trigger" does not exist, skipping - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test2 -ERROR: permission denied for table test2_tablesas -ERROR: permission denied for view test2_view -ERROR: permission denied for materialized view test2_mview -ERROR: permission denied for table test2_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test3 -ERROR: permission denied for table test3_tablesas -ERROR: permission denied for view test3_view -ERROR: permission denied for materialized view test3_mview -ERROR: permission denied for table test3_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - From 164fba197f54d5ca125cf0690d0b1d3021a65622 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 24 Oct 2019 12:13:51 +0200 Subject: [PATCH 24/25] test_foreign_tables: Better debugging and query the default database when needed --- test/extension/test.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/extension/test.sh b/test/extension/test.sh index d23b4b0..faf3d93 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -17,6 +17,10 @@ SED=sed OK=0 PARTIALOK=0 +function reset_default_database() { + DATABASE=test_extension +} + function set_failed() { OK=1 PARTIALOK=1 @@ -40,10 +44,10 @@ function sql() { fi if [ -n "${ROLE}" ]; then - log_debug "Executing query '${QUERY}' as ${ROLE}" + log_debug "Executing query '${QUERY}' as '${ROLE}' in '${DATABASE}'" RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t` else - log_debug "Executing query '${QUERY}'" + log_debug "Executing query '${QUERY}' in '${DATABASE}'" RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t` fi CODERESULT=$? @@ -212,6 +216,8 @@ function tear_down_database() { ${CMD} -c "DROP DATABASE ${DATABASE}" } function tear_down() { + reset_default_database + log_info "########################### USER TEAR DOWN ###########################" sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');" sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');" @@ -532,6 +538,8 @@ END DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);" DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);" + reset_default_database + # Add PGPORT to conf if it is set PORT_SPEC="" if [[ "$PGPORT" != "" ]] ; then @@ -659,8 +667,10 @@ EOF DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;' DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;' + reset_default_database sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';" DATABASE=fdw_target tear_down_database + reset_default_database } function test_cdb_catalog_basic_node() { From 1dcc23db15e95d542ff55066b259d5e1242da6b2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 24 Oct 2019 13:45:36 +0200 Subject: [PATCH 25/25] CDB_UserTablesTest: Make tests work even if the user doesn't have cartodb in the searchpath --- test/CDB_UserTablesTest.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/CDB_UserTablesTest.sql b/test/CDB_UserTablesTest.sql index 3e16c1b..101669d 100644 --- a/test/CDB_UserTablesTest.sql +++ b/test/CDB_UserTablesTest.sql @@ -17,16 +17,16 @@ CREATE TABLE pub(a int); CREATE TABLE prv(a int); GRANT SELECT ON TABLE pub TO publicuser; REVOKE SELECT ON TABLE prv FROM publicuser; -SELECT CDB_UserTables() ORDER BY 1; -SELECT 'all',CDB_UserTables('all') ORDER BY 2; -SELECT 'public',CDB_UserTables('public') ORDER BY 2; -SELECT 'private',CDB_UserTables('private') ORDER BY 2; -SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2; +SELECT cartodb.CDB_UserTables() ORDER BY 1; +SELECT 'all', cartodb.CDB_UserTables('all') ORDER BY 2; +SELECT 'public', cartodb.CDB_UserTables('public') ORDER BY 2; +SELECT 'private', cartodb.CDB_UserTables('private') ORDER BY 2; +SELECT '--unsupported--', cartodb.CDB_UserTables('--unsupported--') ORDER BY 2; -- now tests with public user \c contrib_regression publicuser -SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2; -SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2; -SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2; +SELECT 'all_publicuser', cartodb.CDB_UserTables('all') ORDER BY 2; +SELECT 'public_publicuser', cartodb.CDB_UserTables('public') ORDER BY 2; +SELECT 'private_publicuser', cartodb.CDB_UserTables('private') ORDER BY 2; \c contrib_regression postgres DROP TABLE pub; DROP TABLE prv;