diff --git a/.travis.yml b/.travis.yml index f448081..729d84f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,26 +8,28 @@ env: - PGDATABASE=postgres - PGOPTIONS='-c client_min_messages=NOTICE' - PGPORT=5432 - - POSTGIS_VERSION="2.5" - matrix: - - POSTGRESQL_VERSION="9.6" - - POSTGRESQL_VERSION="10" - - POSTGRESQL_VERSION="11" +jobs: + include: + - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" + - -before_install: +script: - sudo service postgresql stop; - sudo apt-get remove postgresql* -y - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; - - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION + - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis + # For pre12, install plpython2. For PG12 install plpython3 + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi; - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- --auth-local trust --auth-host password - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - - sudo pip install redis==2.4.9 -script: - make - sudo make install - make installcheck diff --git a/Makefile b/Makefile index f59fa62..db588aa 100644 --- a/Makefile +++ b/Makefile @@ -134,12 +134,18 @@ PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +endif $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@ cat $(CDBSCRIPTS) | \ $(SED) -e 's/@extschema@/cartodb/g' \ - -e "s/@postgisschema@/public/g" >> $@ + -e 's/@postgisschema@/public/g' \ + -e 's/plpythonu/$(PLPYTHONU)/g' >> $@ echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@ cat cartodb_version.sql >> $@ @@ -153,10 +159,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION). cp $< $@ $(EXTENSION).control: $(EXTENSION).control.in Makefile - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ # Needed for consistent `echo` results with backslashes SHELL = bash @@ -176,11 +182,7 @@ legacy_regress: $(REGRESS_OLD) Makefile $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" >> $${of}; \ exp=expected/test/$${tn}.out; \ echo '\set ECHO none' > $${exp}; \ - if [[ -f "test/$${tn}_expect.pg$(PG_VERSION)" ]]; then \ - cat test/$${tn}_expect.pg$(PG_VERSION) >> $${exp}; \ - else \ - cat test/$${tn}_expect >> $${exp}; \ - fi \ + cat test/$${tn}_expect >> $${exp}; \ done test_organization: diff --git a/NEWS.md b/NEWS.md index 0ebb16e..2ccee9e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.32.0 (XXXX-XX-XX) +* Some fixes for PG12. +* Make PG12 depend on plpython3u instead of plpythonu +* CDB_UserDataSize is now compatible with postgis 3 without postgis_raster. + 0.31.0 (2019-10-08) * Ghost tables: Add missing tags (#370) * Set search_path in security definer functions. diff --git a/README.md b/README.md index a871807..b4a7b8b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ See [the cartodb-postgresql wiki](https://github.com/CartoDB/cartodb-postgresql/ Dependencies ------------ - * PostgreSQL 9.6+ (with plpythonu extension and xml support) + * PostgreSQL 9.6+ (with plpythonu extension and xml support). For PostgreSQL 12+ plpython3u is required instead of plpythonu. * [PostGIS extension](http://postgis.net) * Python with [Redis module](https://pypi.org/project/redis/) diff --git a/expected/test_setup.out b/expected/test_setup.out index cb4d895..6cea6c0 100644 --- a/expected/test_setup.out +++ b/expected/test_setup.out @@ -1,6 +1,5 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; +SET client_min_messages TO error; +CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ BEGIN diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index eda868a..660d4e8 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -33,9 +33,12 @@ FUNCTION @extschema@._CDB_Group_DropGroup_API(group_name text) RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(group_name)) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url plpy.execute(query) @@ -50,9 +53,12 @@ FUNCTION @extschema@._CDB_Group_RenameGroup_API(old_group_name text, new_group_n RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name)) + url = '/api/v1/databases/{0}/groups/%s' % (pathname2url(old_group_name)) body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role) query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -67,9 +73,12 @@ FUNCTION @extschema@._CDB_Group_AddUsers_API(group_name text, usernames text[]) RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name)) body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -84,9 +93,12 @@ FUNCTION @extschema@._CDB_Group_RemoveUsers_API(group_name text, usernames text[ RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name)) + url = '/api/v1/databases/{0}/groups/%s/users' % (pathname2url(group_name)) body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body) plpy.execute(query) @@ -109,9 +121,12 @@ FUNCTION @extschema@._CDB_Group_Table_GrantPermission_API(group_name text, usern RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name) + url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name) body = '{ "access": "%s" }' % access query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) @@ -134,9 +149,12 @@ FUNCTION @extschema@._CDB_Group_Table_RevokeAllPermission_API(group_name text, u RETURNS VOID AS $$ import string - import urllib + try: + from urllib import pathname2url + except: + from urllib.request import pathname2url - url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name) + url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url plpy.execute(query) $$ LANGUAGE 'plpythonu' @@ -189,7 +207,10 @@ CREATE OR REPLACE FUNCTION @extschema@._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[]) RETURNS int AS $$ - import httplib + try: + import httplib as client + except: + from http import client params = plpy.execute("select c.host, c.port, c.timeout, c.auth from @extschema@._CDB_Group_API_Conf() c;")[0] if params['host'] is None: @@ -202,17 +223,17 @@ $$ last_err = None while retry > 0: try: - client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout']) + conn = SD['groups_api_client'] = client.HTTPConnection(params['host'], params['port'], False, params['timeout']) database_name = plpy.execute("select current_database();")[0]['current_database'] - client.request(method, url.format(database_name), body, headers) - response = client.getresponse() + conn.request(method, url.format(database_name), body, headers) + response = conn.getresponse() assert response.status in valid_return_codes return response.status except Exception as err: retry -= 1 last_err = err plpy.warning('Retrying after: ' + str(err)) - client = SD['groups_api_client'] = None + conn = SD['groups_api_client'] = None if last_err is not None: plpy.error('Fatal Group API error: ' + str(last_err)) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index ec4d307..c73d873 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -397,7 +397,7 @@ DECLARE attr_list TEXT; BEGIN SELECT string_agg(s.c, ',') FROM ( - SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c + SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c ) AS s INTO attr_list; RETURN attr_list; @@ -554,7 +554,7 @@ DECLARE BEGIN SELECT string_agg(@extschema@._CDB_Attribute_Aggregation_Expression(reloid, s.c, table_alias) || Format(' AS %s', s.c), ',') FROM ( - SELECT * FROM @extschema@._CDB_Aggregable_Attributes(reloid) c + SELECT @extschema@._CDB_Aggregable_Attributes(reloid)::text c ) AS s INTO attr_list; RETURN attr_list; @@ -662,7 +662,7 @@ AS $$ offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m); END IF; - point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y); + point_geom := Format('@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y); -- compute the resulting columns in the same order as in the base table WITH cols AS ( @@ -673,7 +673,7 @@ AS $$ Format('@postgisschema@.ST_Transform(%s, 4326) AS the_geom', point_geom) WHEN 'the_geom_webmercator' THEN Format('%s AS the_geom_webmercator', point_geom) - ELSE c + ELSE c::text END AS column FROM @extschema@.CDB_ColumnNames(reloid) c ) @@ -800,7 +800,7 @@ AS $$ '@postgisschema@.ST_Transform(@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom' WHEN 'the_geom_webmercator' THEN '@postgisschema@.ST_SetSRID(@postgisschema@.ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator' - ELSE c + ELSE c::text END AS column FROM CDB_ColumnNames(reloid) c ) @@ -924,7 +924,7 @@ AS $$ SELECT CASE c WHEN 'cartodb_id' THEN 'cartodb_id' - ELSE c + ELSE c::text END AS column FROM @extschema@.CDB_ColumnNames(reloid) c ) diff --git a/scripts-available/CDB_Quota.sql b/scripts-available/CDB_Quota.sql index d3dc983..96b41f2 100644 --- a/scripts-available/CDB_Quota.sql +++ b/scripts-available/CDB_Quota.sql @@ -20,33 +20,50 @@ RETURNS bigint AS $$ DECLARE total_size INT8; + raster_available BOOLEAN; + raster_read_query TEXT; BEGIN + -- Postgis 3+ might not install raster + raster_available := EXISTS ( + SELECT 1 + FROM pg_views + WHERE schemaname = '@postgisschema@' + AND viewname = 'raster_overviews' + ); + + IF raster_available THEN + raster_read_query := Format('SELECT o_table_name, r_table_name FROM @postgisschema@.raster_overviews + WHERE o_table_schema = ''%I'' AND o_table_catalog = current_database()', schema_name); + ELSE + raster_read_query := 'SELECT NULL::text AS o_table_name, NULL::text AS r_table_name'; + END IF; + EXECUTE Format(' WITH raster_tables AS ( - SELECT o_table_name, r_table_name FROM raster_overviews - WHERE o_table_schema = schema_name AND o_table_catalog = current_database() + %s ), user_tables AS ( - SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(schema_name) + SELECT table_name FROM @extschema@._CDB_NonAnalysisTablesInSchema(''%I'') ), table_cat AS ( SELECT table_name, ( EXISTS(select * from raster_tables where o_table_name = table_name) - OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || '[\w\d]*' + OR table_name SIMILAR TO @extschema@._CDB_OverviewTableDiscriminator() || ''[\w\d]*'' ) AS is_overview, EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster FROM user_tables ), sizes AS ( - SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(schema_name, table_name)))) table_size, + SELECT COALESCE(INT8(SUM(@extschema@._CDB_total_relation_size(''%I'', table_name)))) table_size, CASE WHEN is_overview THEN 0 WHEN is_raster THEN 1 ELSE 0.5 -- Division by 2 is for not counting the_geom_webmercator END AS multiplier FROM table_cat GROUP BY is_overview, is_raster ) - SELECT sum(table_size*multiplier)::int8 INTO total_size FROM sizes; + SELECT sum(table_size*multiplier)::int8 FROM sizes + ', raster_read_query, schema_name, schema_name) INTO total_size; IF total_size IS NOT NULL THEN RETURN total_size; diff --git a/sql/test_setup.sql b/sql/test_setup.sql index cb4d895..6cea6c0 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -1,6 +1,5 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; +SET client_min_messages TO error; +CREATE EXTENSION cartodb CASCADE; CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ BEGIN diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 01156f7..469edeb 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -4,7 +4,7 @@ WARNING: Invalidation service configuration not found. Skipping Ghost Tables li NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known. +WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER BEGIN @@ -12,7 +12,7 @@ cdb_ddl_execution 0 CREATE TABLE 1 -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known. +WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE COMMIT diff --git a/test/CDB_OAuth.sql b/test/CDB_OAuth.sql index c3c14e6..3a32b3b 100644 --- a/test/CDB_OAuth.sql +++ b/test/CDB_OAuth.sql @@ -1,6 +1,23 @@ -- Create user and enable OAuth event trigger \set QUIET on SET client_min_messages TO error; + +-- The permission error changed between pre PG11 and post 11 (before everythin "relation", now it's "view", "table" and so on +CREATE OR REPLACE FUNCTION catch_permission_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN insufficient_privilege THEN + RETURN TRUE; + WHEN OTHERS THEN + RAISE WARNING 'Exception %', sqlstate; + RETURN FALSE; +END +$$ LANGUAGE 'plpgsql'; + DROP ROLE IF EXISTS "creator_role"; CREATE ROLE "creator_role" LOGIN; DROP ROLE IF EXISTS "ownership_role"; @@ -30,11 +47,11 @@ SELECT * FROM test_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test; -SELECT * FROM test_tablesas; -SELECT * FROM test_view; -SELECT * FROM test_mview; -SELECT * FROM test_selectinto; +SELECT 'denied_table', catch_permission_error($$SELECT * FROM test;$$); +SELECT 'denied_tableas', catch_permission_error($$SELECT * FROM test_tablesas;$$); +SELECT 'denied_view', catch_permission_error($$SELECT * FROM test_view;$$); +SELECT 'denied_mview', catch_permission_error($$SELECT * FROM test_mview;$$); +SELECT 'denied_selectinto', catch_permission_error($$SELECT * FROM test_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -71,11 +88,11 @@ SELECT * FROM test2_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test2; -SELECT * FROM test2_tablesas; -SELECT * FROM test2_view; -SELECT * FROM test2_mview; -SELECT * FROM test2_selectinto; +SELECT 'denied_table2', catch_permission_error($$SELECT * FROM test2;$$); +SELECT 'denied_tableas2', catch_permission_error($$SELECT * FROM test2_tablesas;$$); +SELECT 'denied_view2', catch_permission_error($$SELECT * FROM test2_view;$$); +SELECT 'denied_mview2', catch_permission_error($$SELECT * FROM test2_mview;$$); +SELECT 'denied_selectinto2', catch_permission_error($$SELECT * FROM test2_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -112,11 +129,11 @@ SELECT * FROM test3_selectinto; SET SESSION AUTHORIZATION "ownership_role"; \set QUIET off -SELECT * FROM test3; -SELECT * FROM test3_tablesas; -SELECT * FROM test3_view; -SELECT * FROM test3_mview; -SELECT * FROM test3_selectinto; +SELECT 'denied_table3', catch_permission_error($$SELECT * FROM test3;$$); +SELECT 'denied_tableas3', catch_permission_error($$SELECT * FROM test3_tablesas;$$); +SELECT 'denied_view3', catch_permission_error($$SELECT * FROM test3_view;$$); +SELECT 'denied_mview3', catch_permission_error($$SELECT * FROM test3_mview;$$); +SELECT 'denied_selectinto3', catch_permission_error($$SELECT * FROM test3_selectinto;$$); \set QUIET on SET SESSION AUTHORIZATION "creator_role"; @@ -174,4 +191,5 @@ DROP ROLE "ownership_role"; REVOKE ALL ON SCHEMA cartodb FROM "creator_role"; DROP ROLE "creator_role"; DELETE FROM cdb_conf WHERE key = 'api_keys_creator_role'; +DROP FUNCTION catch_permission_error(text); \set QUIET off diff --git a/test/CDB_OAuth_expect b/test/CDB_OAuth_expect index f536c43..1a334c0 100644 --- a/test/CDB_OAuth_expect +++ b/test/CDB_OAuth_expect @@ -10,11 +10,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test -ERROR: permission denied for relation test_tablesas -ERROR: permission denied for relation test_view -ERROR: permission denied for relation test_mview -ERROR: permission denied for relation test_selectinto +denied_table|t +denied_tableas|t +denied_view|t +denied_mview|t +denied_selectinto|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW @@ -33,11 +33,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test2 -ERROR: permission denied for relation test2_tablesas -ERROR: permission denied for relation test2_view -ERROR: permission denied for relation test2_mview -ERROR: permission denied for relation test2_selectinto +denied_table2|t +denied_tableas2|t +denied_view2|t +denied_mview2|t +denied_selectinto2|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW @@ -55,11 +55,11 @@ SELECT 1 1 1 1 -ERROR: permission denied for relation test3 -ERROR: permission denied for relation test3_tablesas -ERROR: permission denied for relation test3_view -ERROR: permission denied for relation test3_mview -ERROR: permission denied for relation test3_selectinto +denied_table3|t +denied_tableas3|t +denied_view3|t +denied_mview3|t +denied_selectinto3|t DROP TABLE DROP VIEW DROP MATERIALIZED VIEW diff --git a/test/CDB_OAuth_expect.pg11 b/test/CDB_OAuth_expect.pg11 deleted file mode 100644 index 73767d2..0000000 --- a/test/CDB_OAuth_expect.pg11 +++ /dev/null @@ -1,90 +0,0 @@ - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test -ERROR: permission denied for table test_tablesas -ERROR: permission denied for view test_view -ERROR: permission denied for materialized view test_mview -ERROR: permission denied for table test_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE -NOTICE: event trigger "oauth_reassign_tables_trigger" does not exist, skipping - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test2 -ERROR: permission denied for table test2_tablesas -ERROR: permission denied for view test2_view -ERROR: permission denied for materialized view test2_mview -ERROR: permission denied for table test2_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -ERROR: permission denied for table test3 -ERROR: permission denied for table test3_tablesas -ERROR: permission denied for view test3_view -ERROR: permission denied for materialized view test3_mview -ERROR: permission denied for table test3_selectinto -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - -CREATE TABLE -INSERT 0 1 -SELECT 1 -CREATE VIEW -SELECT 1 -SELECT 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -DROP TABLE -DROP VIEW -DROP MATERIALIZED VIEW -DROP TABLE -DROP TABLE - diff --git a/test/CDB_QuotaTest.sql b/test/CDB_QuotaTest.sql index 92b65d1..bbb8753 100644 --- a/test/CDB_QuotaTest.sql +++ b/test/CDB_QuotaTest.sql @@ -1,30 +1,48 @@ set client_min_messages to error; \set VERBOSITY TERSE - --- See the dice -SELECT setseed(0.5); +-- Runs a query and returns whether an error was thrown +-- Useful when the error message depends on the execution plan or db settings +-- The error message outputs the extra quota, and this might depend on the database setup and version +CREATE OR REPLACE FUNCTION catch_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN OTHERS THEN + RETURN TRUE; +END +$$ LANGUAGE 'plpgsql'; CREATE TABLE big(a int); -- Try the legacy interface -- See https://github.com/CartoDB/cartodb-postgresql/issues/13 CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big - EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public'); + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); INSERT INTO big VALUES (1); -- allowed, check runs before -INSERT INTO big VALUES (2); -- disallowed, quota exceeds before -SELECT CDB_SetUserQuotaInBytes(0); -SELECT CDB_CartodbfyTable('big'); +SELECT 'excess1', catch_error($$INSERT INTO big VALUES (2); $$); -- disallowed, quota exceeds before +SELECT cartodb.CDB_SetUserQuotaInBytes(0); +SELECT cartodb.CDB_CartodbfyTable('big'); +-- Creating the trigger should fail as it was created by CDB_CartodbfyTable +CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); +-- Drop the trigger and recreate it forcing a 100% checks +DROP TRIGGER test_quota ON big; +CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); INSERT INTO big SELECT generate_series(2049,4096); INSERT INTO big SELECT generate_series(4097,6144); INSERT INTO big SELECT generate_series(6145,8192); -- Test for #108: https://github.com/CartoDB/cartodb-postgresql/issues/108 -SELECT CDB_UserDataSize(); -SELECT cartodb._CDB_total_relation_size('public', 'big'); +SELECT cartodb.CDB_UserDataSize() < 500000 AND cartodb.CDB_UserDataSize() > 0; +SELECT cartodb._CDB_total_relation_size('public', 'big') < 1000000; SELECT cartodb._CDB_total_relation_size('public', 'nonexistent_table_name'); -- END Test for #108 -SELECT setseed(0.9); -SELECT CDB_SetUserQuotaInBytes(2); -INSERT INTO big VALUES (8193); -SELECT CDB_SetUserQuotaInBytes(0); + +SELECT cartodb.CDB_SetUserQuotaInBytes(2); +SELECT 'excess2', catch_error($$INSERT INTO big VALUES (8193);$$); +SELECT cartodb.CDB_SetUserQuotaInBytes(0); INSERT INTO big VALUES (8194); DROP TABLE big; @@ -32,16 +50,17 @@ DROP TABLE big; --analysis tables should be excluded from quota: CREATE TABLE big(a int); CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big - EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public'); -SELECT CDB_SetUserQuotaInBytes(1); + EXECUTE PROCEDURE cartodb.CDB_CheckQuota(2, 1, 'public'); +SELECT cartodb.CDB_SetUserQuotaInBytes(1); CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int); INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5); INSERT INTO big VALUES (1); -- allowed, check runs before DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4; -INSERT INTO big VALUES (2); -- disallowed, quota exceeds before +SELECT 'excess3', catch_error($$INSERT INTO big VALUES (3);$$); -- disallowed, quota exceeds before DROP TABLE big; SELECT CDB_SetUserQuotaInBytes(0); set client_min_messages to NOTICE; +DROP FUNCTION catch_error(text); DROP FUNCTION _CDB_UserQuotaInBytes(); diff --git a/test/CDB_QuotaTest_expect b/test/CDB_QuotaTest_expect index 9204514..49f9e7d 100644 --- a/test/CDB_QuotaTest_expect +++ b/test/CDB_QuotaTest_expect @@ -1,20 +1,22 @@ SET - +CREATE FUNCTION CREATE TABLE CREATE TRIGGER INSERT 0 1 -ERROR: Quota exceeded by 3.9990234375KB +excess1|t 0 big +ERROR: trigger "test_quota" for relation "big" already exists +DROP TRIGGER +CREATE TRIGGER INSERT 0 2048 INSERT 0 2048 INSERT 0 2048 -454656 -909312 +t +t 0 - 2 -ERROR: Quota exceeded by 443.998046875KB +excess2|t 0 INSERT 0 1 DROP TABLE @@ -25,8 +27,9 @@ CREATE TABLE INSERT 0 5 INSERT 0 1 DROP TABLE -ERROR: Quota exceeded by 3.9990234375KB +excess3|t DROP TABLE 0 SET DROP FUNCTION +DROP FUNCTION diff --git a/test/CDB_UserTablesTest.sql b/test/CDB_UserTablesTest.sql index 3e16c1b..101669d 100644 --- a/test/CDB_UserTablesTest.sql +++ b/test/CDB_UserTablesTest.sql @@ -17,16 +17,16 @@ CREATE TABLE pub(a int); CREATE TABLE prv(a int); GRANT SELECT ON TABLE pub TO publicuser; REVOKE SELECT ON TABLE prv FROM publicuser; -SELECT CDB_UserTables() ORDER BY 1; -SELECT 'all',CDB_UserTables('all') ORDER BY 2; -SELECT 'public',CDB_UserTables('public') ORDER BY 2; -SELECT 'private',CDB_UserTables('private') ORDER BY 2; -SELECT '--unsupported--',CDB_UserTables('--unsupported--') ORDER BY 2; +SELECT cartodb.CDB_UserTables() ORDER BY 1; +SELECT 'all', cartodb.CDB_UserTables('all') ORDER BY 2; +SELECT 'public', cartodb.CDB_UserTables('public') ORDER BY 2; +SELECT 'private', cartodb.CDB_UserTables('private') ORDER BY 2; +SELECT '--unsupported--', cartodb.CDB_UserTables('--unsupported--') ORDER BY 2; -- now tests with public user \c contrib_regression publicuser -SELECT 'all_publicuser',CDB_UserTables('all') ORDER BY 2; -SELECT 'public_publicuser',CDB_UserTables('public') ORDER BY 2; -SELECT 'private_publicuser',CDB_UserTables('private') ORDER BY 2; +SELECT 'all_publicuser', cartodb.CDB_UserTables('all') ORDER BY 2; +SELECT 'public_publicuser', cartodb.CDB_UserTables('public') ORDER BY 2; +SELECT 'private_publicuser', cartodb.CDB_UserTables('private') ORDER BY 2; \c contrib_regression postgres DROP TABLE pub; DROP TABLE prv; diff --git a/test/extension/test.sh b/test/extension/test.sh index d23b4b0..faf3d93 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -17,6 +17,10 @@ SED=sed OK=0 PARTIALOK=0 +function reset_default_database() { + DATABASE=test_extension +} + function set_failed() { OK=1 PARTIALOK=1 @@ -40,10 +44,10 @@ function sql() { fi if [ -n "${ROLE}" ]; then - log_debug "Executing query '${QUERY}' as ${ROLE}" + log_debug "Executing query '${QUERY}' as '${ROLE}' in '${DATABASE}'" RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t` else - log_debug "Executing query '${QUERY}'" + log_debug "Executing query '${QUERY}' in '${DATABASE}'" RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t` fi CODERESULT=$? @@ -212,6 +216,8 @@ function tear_down_database() { ${CMD} -c "DROP DATABASE ${DATABASE}" } function tear_down() { + reset_default_database + log_info "########################### USER TEAR DOWN ###########################" sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');" sql cdb_testmember_2 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_2', 'bar', 'cdb_testmember_1');" @@ -532,6 +538,8 @@ END DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo'::regclass);" DATABASE=fdw_target sql postgres "SELECT cdb_tablemetadatatouch('test_fdw.foo2'::regclass);" + reset_default_database + # Add PGPORT to conf if it is set PORT_SPEC="" if [[ "$PGPORT" != "" ]] ; then @@ -659,8 +667,10 @@ EOF DATABASE=fdw_target sql postgres 'REVOKE SELECT ON cdb_tablemetadata_text FROM fdw_user;' DATABASE=fdw_target sql postgres 'DROP ROLE fdw_user;' + reset_default_database sql postgres "select pg_terminate_backend(pid) from pg_stat_activity where datname='fdw_target';" DATABASE=fdw_target tear_down_database + reset_default_database } function test_cdb_catalog_basic_node() {