From b475eb435619a97ee183fd716720e682433e07ae Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 28 Oct 2019 14:08:07 +0100 Subject: [PATCH 001/108] Federated Server List --- scripts-available/CDB_FederatedServer.sql | 70 +++++++++++++++++++++ scripts-enabled/400-CDB_FederatedServer.sql | 1 + test/CDB_FederatedServer.sql | 43 +++++++++++++ test/CDB_FederatedServer_expect | 0 4 files changed, 114 insertions(+) create mode 100644 scripts-available/CDB_FederatedServer.sql create mode 120000 scripts-enabled/400-CDB_FederatedServer.sql create mode 100644 test/CDB_FederatedServer.sql create mode 100644 test/CDB_FederatedServer_expect diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql new file mode 100644 index 0000000..46af6d8 --- /dev/null +++ b/scripts-available/CDB_FederatedServer.sql @@ -0,0 +1,70 @@ + +-- This function is just a placement to store and use the pattern for +-- foreign server names +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() +RETURNS TEXT +AS $$ + SELECT 'cdb_fs_'; +$$ +LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +-- Produce a valid DB name for objects created for the user FDW's +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Object_Name(fdw_input_name NAME) +RETURNS NAME +AS $$ +DECLARE + object_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), fdw_input_name); +BEGIN + -- We discard anything that would be truncated + IF (char_length(object_name) < 64) THEN + RETURN object_name::name; + ELSE + RAISE EXCEPTION 'Object name is too long to be used as identifier'; + END IF; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + + + +-- List registered servers +-- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate)s +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(fdw_pattern TEXT DEFAULT '%') +RETURNS TABLE ( + name text, + driver text, + host text, + port text, + dbname text, + readmode text, + username text +) +AS $$ +DECLARE + server_name text := concat(@extschema@.__CDB_FS_Name_Pattern() || fdw_pattern); +BEGIN + RETURN QUERY SELECT + -- Name as shown to the user + right(s.srvname, char_length(s.srvname::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT AS "Name", + + -- Which driver are we using (postgres_fdw, odbc_fdw...) + f.fdwname::text AS "Driver", + + -- Read + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", + CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", + + -- Read username from user mappings + (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" + FROM pg_foreign_server s + JOIN pg_foreign_data_wrapper f ON f.oid=s.srvfdw + LEFT JOIN pg_user_mappings u + ON u.srvid = s.oid + WHERE s.srvname ILIKE server_name + ORDER BY 1; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; diff --git a/scripts-enabled/400-CDB_FederatedServer.sql b/scripts-enabled/400-CDB_FederatedServer.sql new file mode 120000 index 0000000..6a3df9a --- /dev/null +++ b/scripts-enabled/400-CDB_FederatedServer.sql @@ -0,0 +1 @@ +../scripts-available/CDB_FederatedServer.sql \ No newline at end of file diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql new file mode 100644 index 0000000..dd25170 --- /dev/null +++ b/test/CDB_FederatedServer.sql @@ -0,0 +1,43 @@ +-- Setup +\set QUIET on +SET client_min_messages TO error; +SET SESSION AUTHORIZATION postgres; +\set QUIET off + +-- Register a new server +SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); +-- SELECT '1.2', cartodb.CDB_Federated_Server_Register_PG(); +-- SELECT '1.3', cartodb.CDB_Federated_Server_List_Servers(); +-- +-- Register a second server +-- SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(); +-- SELECT '2.2', cartodb.CDB_PG_Federated_Server_List_Servers(); +-- +-- Re-register the second server +-- SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(); +-- SELECT '3.2', cartodb.CDB_PG_Federated_Server_List_Servers(); +-- +-- Unregister #1 +-- SELECT '4.1', cartodb.CDB_PG_Federated_Server_Unregister(); +-- SELECT '4.2', cartodb.CDB_PG_Federated_Server_List_Servers(); +-- +-- Unregister #2 +-- SELECT '5.1', cartodb.CDB_PG_Federated_Server_Unregister(); +-- SELECT '5.2', cartodb.CDB_PG_Federated_Server_List_Servers(); + +-- Should show the appropiate output (database, read-write, user, pass) + +-- Test multiple user mappings + +-- Should work ok with special characters in the name + +-- Should throw with invalid or NULL config + +-- -- List works with multiple and single server + +-- Cleanup +\set QUIET on + +\set QUIET off + + diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect new file mode 100644 index 0000000..e69de29 From eb02ca4d86ebc362f82ffcaac6159c4f876d5d81 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 28 Oct 2019 17:09:51 +0100 Subject: [PATCH 002/108] Regress: Allow variables in the tests --- Makefile | 30 +++++++++++++++++++++++++----- sql/test_setup.sql | 2 +- test/CDB_Username_expect | 2 +- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 99682e4..c62a95b 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,8 @@ PLPYTHONU := plpythonu ifeq ($(PG_12_GE), true) PLPYTHONU := plpython3u endif +PGPORT ?= '5432' +PGUSER ?= 'postgres' $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@ @@ -172,7 +174,7 @@ legacy_regress: $(REGRESS_OLD) Makefile mkdir -p expected/test/ mkdir -p results/test/ cat sql/test_setup.sql | \ - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' \ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ > sql/test/test_setup.sql cp sql/test_setup_expect expected/test/test_setup.out for f in $(REGRESS_OLD); do \ @@ -183,10 +185,23 @@ legacy_regress: $(REGRESS_OLD) Makefile echo '\t' >> $${of}; \ echo '\set QUIET off' >> $${of}; \ cat $${f} | \ - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' >> $${of}; \ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' \ + -e 's/@extschema@/cartodb/g' \ + -e "s/@postgisschema@/public/g" \ + -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + -e 's/@@PGPORT@@/$(PGPORT)/g' \ + -e 's/@@PGUSER@@/$(PGUSER)/g' \ + >> $${of}; \ exp=expected/test/$${tn}.out; \ echo '\set ECHO none' > $${exp}; \ - cat test/$${tn}_expect >> $${exp}; \ + cat test/$${tn}_expect | \ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' \ + -e 's/@extschema@/cartodb/g' \ + -e "s/@postgisschema@/public/g" \ + -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \ + -e 's/@@PGPORT@@/$(PGPORT)/g' \ + -e 's/@@PGUSER@@/$(PGUSER)/g' \ + >> $${exp}; \ done test_organization: @@ -198,8 +213,13 @@ test_extension_new: legacy_tests: legacy_regress $(EXTENSION)--unpackaged--$(EXTVERSION).sql PGREGRESS := $(shell dirname `$(PG_CONFIG) --pgxs`)/../../src/test/regress/pg_regress +PGBINDIR := $(shell $(PG_CONFIG) --bindir) +PGDATABASE = 'contrib_regression' regress: legacy_tests - $(PGREGRESS) --inputdir=./ --bindir='/usr/bin' --dbname=contrib_regression $(REGRESS) + PGUSER=$(PGUSER) \ + PGPORT=$(PGPORT) \ + $(PGREGRESS) --inputdir=./ --bindir='$(PGBINDIR)' --dbname=$(PGDATABASE) $(REGRESS) -installcheck: legacy_tests test_extension_new test_organization +installcheck: test_extension_new test_organization + $(MAKE) -C regress diff --git a/sql/test_setup.sql b/sql/test_setup.sql index ec5021b..cf55bf4 100644 --- a/sql/test_setup.sql +++ b/sql/test_setup.sql @@ -2,7 +2,7 @@ \set QUIET on SET client_min_messages TO error; CREATE EXTENSION postgis; -CREATE EXTENSION plpythonu; +CREATE EXTENSION @@plpythonu@@; CREATE SCHEMA cartodb; \i 'cartodb--unpackaged--@@VERSION@@.sql' CREATE FUNCTION public.cdb_invalidate_varnish(table_name text) diff --git a/test/CDB_Username_expect b/test/CDB_Username_expect index fe1ea5b..d030a42 100644 --- a/test/CDB_Username_expect +++ b/test/CDB_Username_expect @@ -1,4 +1,4 @@ -postgres +@@PGUSER@@ fulano fulanito From acf1f2dbd4458ea1dab6ebb572fd7ea3a30155ac Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 28 Oct 2019 18:34:46 +0100 Subject: [PATCH 003/108] WIP: Add CDB_Federated_Server_Unregister --- scripts-available/CDB_FederatedServer.sql | 85 +++++++++++++++++++++-- scripts-available/CDB_ForeignTable.sql | 6 +- test/CDB_FederatedServer.sql | 56 +++++++++++---- test/CDB_FederatedServer_expect | 12 ++++ 4 files changed, 137 insertions(+), 22 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 46af6d8..a5137c7 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -10,7 +10,7 @@ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- Produce a valid DB name for objects created for the user FDW's -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Object_Name(fdw_input_name NAME) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Object_Name(fdw_input_name NAME) RETURNS NAME AS $$ DECLARE @@ -26,11 +26,19 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Public_Name(fdw_stored_name NAME) +RETURNS TEXT +AS $$ + SELECT right(fdw_stored_name, + char_length(fdw_stored_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; +$$ +LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- List registered servers --- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate)s -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(fdw_pattern TEXT DEFAULT '%') +-- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) +-- TODO: Handle multiple user mappings in the same server +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') RETURNS TABLE ( name text, driver text, @@ -42,22 +50,22 @@ RETURNS TABLE ( ) AS $$ DECLARE - server_name text := concat(@extschema@.__CDB_FS_Name_Pattern() || fdw_pattern); + server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server); BEGIN RETURN QUERY SELECT -- Name as shown to the user - right(s.srvname, char_length(s.srvname::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT AS "Name", + @extschema@.__CDB_FS_Extract_Public_Name(s.srvname) AS "Name", -- Which driver are we using (postgres_fdw, odbc_fdw...) f.fdwname::text AS "Driver", - -- Read + -- Read options from pg_foreign_server (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", - -- Read username from user mappings + -- Read username from pg_user_mappings (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" FROM pg_foreign_server s JOIN pg_foreign_data_wrapper f ON f.oid=s.srvfdw @@ -68,3 +76,66 @@ BEGIN END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Take a config jsonb and transform it to an input suitable for +-- _CDB_SetUp_User_PG_FDW_Server +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB) +RETURNS jsonb +AS $$ +DECLARE + user_mapping jsonb := json_build_object( + 'user_mapping', + jsonb_build_object('user', input_config->'credentials'->'username', + 'password', input_config->'credentials'->'password') + ); +BEGIN + RETURN (input_config - 'credentials')::jsonb || user_mapping; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Take a config jsonb as input and return it augmented with default +-- options +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb) +RETURNS jsonb +AS $$ +DECLARE + default_options jsonb := '{ + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }'; + server_config jsonb; +BEGIN + server_config := default_options || to_jsonb(input_config->'server'); + RETURN jsonb_set(input_config, '{server}'::text[], server_config); +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB) +RETURNS void +AS $$ +DECLARE + -- TODO: Check and handle existing servers (if needed) + final_name text := @extschema@.__CDB_FS_Generate_Object_Name(server); + final_config jsonb := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); +BEGIN + PERFORM @extschema@._CDB_SetUp_User_PG_FDW_Server(final_name, final_config::json); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION cartodb.CDB_Federated_Server_Unregister(server TEXT) +RETURNS void +AS $$ +DECLARE + final_name text := cartodb.__CDB_FS_Generate_Object_Name(server); +BEGIN + -- TODO: Check if the server exists to show a nice error instead of the automatic one (which uses the implementation name) + EXECUTE 'DROP SERVER ' || quote_ident(final_name) || ' CASCADE'; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index c86f5ef..9a772b4 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -184,15 +184,17 @@ RETURNS void AS $$ DECLARE row record; option record; - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); + fdw_objects_name NAME := fdw_input_name; BEGIN -- TODO: refactor with original function -- This function tries to be as idempotent as possible, by not creating anything more than once -- (not even using IF NOT EXIST to avoid throwing warnings) IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN CREATE EXTENSION postgres_fdw; - RAISE NOTICE 'Created postgres_fdw EXTENSION'; + RAISE EXCEPTION 'postgres_fdw extension is not installed' + USING HINT = 'Please install it with "CREATE EXTENSION postgres_fdw"'; END IF; + -- Create FDW first if it does not exist IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name) THEN diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index dd25170..3764c46 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -2,28 +2,58 @@ \set QUIET on SET client_min_messages TO error; SET SESSION AUTHORIZATION postgres; +CREATE EXTENSION postgres_fdw; \set QUIET off -- Register a new server SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); --- SELECT '1.2', cartodb.CDB_Federated_Server_Register_PG(); --- SELECT '1.3', cartodb.CDB_Federated_Server_List_Servers(); --- +SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server := 'doesNotExist'); +SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); + -- Register a second server --- SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(); --- SELECT '2.2', cartodb.CDB_PG_Federated_Server_List_Servers(); --- +SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); + + -- Re-register the second server -- SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(); -- SELECT '3.2', cartodb.CDB_PG_Federated_Server_List_Servers(); --- + -- Unregister #1 --- SELECT '4.1', cartodb.CDB_PG_Federated_Server_Unregister(); --- SELECT '4.2', cartodb.CDB_PG_Federated_Server_List_Servers(); --- +SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote'::text); +SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); + +-- Unregister a server that doesn't exist +SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist'::text); +SELECT '5.2', cartodb.CDB_Federated_Server_List_Servers(); + -- Unregister #2 --- SELECT '5.1', cartodb.CDB_PG_Federated_Server_Unregister(); --- SELECT '5.2', cartodb.CDB_PG_Federated_Server_List_Servers(); +SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); +SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); -- Should show the appropiate output (database, read-write, user, pass) @@ -37,7 +67,7 @@ SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); -- Cleanup \set QUIET on - +DROP EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index e69de29..c142968 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -0,0 +1,12 @@ +1.3| +1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +2.1| +2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +4.1| +4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +ERROR: server "cdb_fs_doesNotExist" does not exist +CONTEXT: SQL statement "DROP SERVER "cdb_fs_doesNotExist" CASCADE" +PL/pgSQL function cdb_federated_server_unregister(text) line 5 at EXECUTE +5.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +6.1| From f285c806fc1657e97aea47d667eaf55a67bfa346 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 11:57:24 +0100 Subject: [PATCH 004/108] Remove error context --- test/CDB_FederatedServer.sql | 1 + test/CDB_FederatedServer_expect | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 3764c46..32d4e7c 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -1,6 +1,7 @@ -- Setup \set QUIET on SET client_min_messages TO error; +\set VERBOSITY terse SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index c142968..c7e8aec 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -6,7 +6,5 @@ 4.1| 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) ERROR: server "cdb_fs_doesNotExist" does not exist -CONTEXT: SQL statement "DROP SERVER "cdb_fs_doesNotExist" CASCADE" -PL/pgSQL function cdb_federated_server_unregister(text) line 5 at EXECUTE 5.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) 6.1| From 1375894e995817ca542ac26f133d987f989b6e50 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 12:24:54 +0100 Subject: [PATCH 005/108] CDB_Federated_Server_Unregister: Better error handling --- scripts-available/CDB_FederatedServer.sql | 12 ++++++++---- scripts-available/CDB_ForeignTable.sql | 2 +- test/CDB_FederatedServer.sql | 2 ++ test/CDB_FederatedServer_expect | 3 ++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index a5137c7..a0e6331 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -128,14 +128,18 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -CREATE OR REPLACE FUNCTION cartodb.CDB_Federated_Server_Unregister(server TEXT) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT) RETURNS void AS $$ DECLARE - final_name text := cartodb.__CDB_FS_Generate_Object_Name(server); + final_name text := @extschema@.__CDB_FS_Generate_Object_Name(server); BEGIN - -- TODO: Check if the server exists to show a nice error instead of the automatic one (which uses the implementation name) - EXECUTE 'DROP SERVER ' || quote_ident(final_name) || ' CASCADE'; + IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = final_name) + THEN + RAISE EXCEPTION 'Server "%" does not exist', server; + END IF; + + EXECUTE @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name := final_name, force := true); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 9a772b4..a258c89 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -271,7 +271,7 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false) RETURNS void AS $$ DECLARE - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); + fdw_objects_name NAME := fdw_input_name; cascade_clause NAME; BEGIN CASE force diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 32d4e7c..a445e73 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -38,6 +38,8 @@ SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te } }'::jsonb); SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); +-- Check that CDB_Federated_Server_List_Servers works with name +SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server := 'myRemote'); -- Re-register the second server diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index c7e8aec..768b271 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -3,8 +3,9 @@ 2.1| 2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 4.1| 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) -ERROR: server "cdb_fs_doesNotExist" does not exist +ERROR: Server "doesNotExist" does not exist 5.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) 6.1| From 427ad942356e5fc273fb4ec99ae9f09f6c5991f2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 12:34:29 +0100 Subject: [PATCH 006/108] Fix installcheck target --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c62a95b..0f902aa 100644 --- a/Makefile +++ b/Makefile @@ -214,12 +214,12 @@ legacy_tests: legacy_regress $(EXTENSION)--unpackaged--$(EXTVERSION).sql PGREGRESS := $(shell dirname `$(PG_CONFIG) --pgxs`)/../../src/test/regress/pg_regress PGBINDIR := $(shell $(PG_CONFIG) --bindir) -PGDATABASE = 'contrib_regression' +PGREGRESSDATABASE = 'contrib_regression' regress: legacy_tests PGUSER=$(PGUSER) \ PGPORT=$(PGPORT) \ - $(PGREGRESS) --inputdir=./ --bindir='$(PGBINDIR)' --dbname=$(PGDATABASE) $(REGRESS) + $(PGREGRESS) --inputdir=./ --bindir='$(PGBINDIR)' --dbname=$(PGREGRESSDATABASE) $(REGRESS) -installcheck: test_extension_new test_organization - $(MAKE) -C regress +installcheck: test_extension_new test_organization regress + $(MAKE) -f ... -C regress From bc7031d872121440459b975b8da52a5b5af23ecf Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 13:19:11 +0100 Subject: [PATCH 007/108] Move existence check to __CDB_FS_Generate_Object_Name --- scripts-available/CDB_FederatedServer.sql | 28 +++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index a0e6331..67d13f4 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -10,18 +10,21 @@ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- Produce a valid DB name for objects created for the user FDW's -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Object_Name(fdw_input_name NAME) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Object_Name(input_name NAME, check_existence BOOL) RETURNS NAME AS $$ DECLARE - object_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), fdw_input_name); + object_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); BEGIN - -- We discard anything that would be truncated - IF (char_length(object_name) < 64) THEN - RETURN object_name::name; - ELSE - RAISE EXCEPTION 'Object name is too long to be used as identifier'; - END IF; + -- We discard anything that would be truncated + IF (char_length(object_name) < 64) THEN + IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = object_name))) THEN + RAISE EXCEPTION 'Server "%" does not exist', input_name; + END IF; + RETURN object_name::name; + ELSE + RAISE EXCEPTION 'Object name is too long to be used as identifier'; + END IF; END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; @@ -120,7 +123,7 @@ RETURNS void AS $$ DECLARE -- TODO: Check and handle existing servers (if needed) - final_name text := @extschema@.__CDB_FS_Generate_Object_Name(server); + final_name text := @extschema@.__CDB_FS_Generate_Object_Name(input_name := server, check_existence := false); final_config jsonb := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); BEGIN PERFORM @extschema@._CDB_SetUp_User_PG_FDW_Server(final_name, final_config::json); @@ -132,13 +135,8 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TE RETURNS void AS $$ DECLARE - final_name text := @extschema@.__CDB_FS_Generate_Object_Name(server); + final_name text := @extschema@.__CDB_FS_Generate_Object_Name(input_name := server, check_existence := true); BEGIN - IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = final_name) - THEN - RAISE EXCEPTION 'Server "%" does not exist', server; - END IF; - EXECUTE @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name := final_name, force := true); END $$ From e6cb03300b0be1c38473c9d92517c3456e8f8da3 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 15:38:58 +0100 Subject: [PATCH 008/108] Add auxiliar functions to handle internal naming --- scripts-available/CDB_FederatedServer.sql | 67 ++++++++++++++++++++--- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 67d13f4..b647992 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -1,16 +1,19 @@ -- This function is just a placement to store and use the pattern for -- foreign server names +-- Servers: cdb_fs_$(server_name) +-- Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) +-- Owner role: cdb_fs_$(md5sum(current_database() || server_name) CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() RETURNS TEXT AS $$ - SELECT 'cdb_fs_'; + SELECT 'cdb_fs_'::text; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; --- Produce a valid DB name for objects created for the user FDW's -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Object_Name(input_name NAME, check_existence BOOL) +-- Produce a valid DB name for servers generated for the Federated Server +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) RETURNS NAME AS $$ DECLARE @@ -23,20 +26,66 @@ BEGIN END IF; RETURN object_name::name; ELSE - RAISE EXCEPTION 'Object name is too long to be used as identifier'; + RAISE EXCEPTION 'Server name is too long to be used as identifier'; END IF; END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Public_Name(fdw_stored_name NAME) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(fdw_stored_name NAME) RETURNS TEXT AS $$ SELECT right(fdw_stored_name, - char_length(fdw_stored_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; + char_length(fdw_stored_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +-- Produce a valid name for a schema generated for the Federated Server +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(server_name TEXT, schema_name TEXT) +RETURNS NAME +AS $$ +DECLARE + server_full_name text := @extschema@.__CDB_FS_Generate_Server_Name(server_name, check_existence := true); + hash_value text := md5(server_full_name::text || '__' || schema_name::text); + schema_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value); +BEGIN + RETURN schema_name::name; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Produce a valid name for a role generated for the Federated Server +-- This needs to include the current database in its hash to avoid collisions in clusters with more than one database +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(server_name TEXT) +RETURNS NAME +AS $$ +DECLARE + server_full_name text := @extschema@.__CDB_FS_Generate_Server_Name(server_name, check_existence := true); + hash_value text := md5(current_database()::text || '__' || server_full_name::text); + role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value); +BEGIN + RETURN role_name::name; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + +-- Creates (if not exist) a schema to place the objects for a remote schema +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(server_name TEXT, schema_name TEXT) +RETURNS NAME +AS $$ +DECLARE + schema_name text := @extschema@.__CDB_FS_Generate_Schema_Name(server_name, schema_name); + role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_name); +BEGIN + BEGIN + EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'TODO: This needs a better error handling after reviewing permissions'; + END; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + -- List registered servers -- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) @@ -57,7 +106,7 @@ DECLARE BEGIN RETURN QUERY SELECT -- Name as shown to the user - @extschema@.__CDB_FS_Extract_Public_Name(s.srvname) AS "Name", + @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", -- Which driver are we using (postgres_fdw, odbc_fdw...) f.fdwname::text AS "Driver", @@ -123,7 +172,7 @@ RETURNS void AS $$ DECLARE -- TODO: Check and handle existing servers (if needed) - final_name text := @extschema@.__CDB_FS_Generate_Object_Name(input_name := server, check_existence := false); + final_name text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); final_config jsonb := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); BEGIN PERFORM @extschema@._CDB_SetUp_User_PG_FDW_Server(final_name, final_config::json); @@ -135,7 +184,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TE RETURNS void AS $$ DECLARE - final_name text := @extschema@.__CDB_FS_Generate_Object_Name(input_name := server, check_existence := true); + final_name text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); BEGIN EXECUTE @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name := final_name, force := true); END From da9ff0c7507d92a2e90ebc6e7b51b691ce4e6664 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 16:19:46 +0100 Subject: [PATCH 009/108] Reimplement CDB_Federated_Server_Register_PG and CDB_Federated_Server_Unregister --- scripts-available/CDB_FederatedServer.sql | 71 +++++++++++++++++++++-- scripts-available/CDB_ForeignTable.sql | 8 +-- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index b647992..d33c7fa 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -172,10 +172,68 @@ RETURNS void AS $$ DECLARE -- TODO: Check and handle existing servers (if needed) - final_name text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); - final_config jsonb := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); + role_name name; + row record; + option record; BEGIN - PERFORM @extschema@._CDB_SetUp_User_PG_FDW_Server(final_name, final_config::json); + IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN + RAISE EXCEPTION 'postgres_fdw extension is not installed' + USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`'; + END IF; + + -- We only create server and roles if the server didn't exist before + IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN + BEGIN + EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); + role_name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server); + EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); + EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); + -- NOTE: we use a PUBLIC user mapping but control access to the SERVER + -- so that we don't need to create a mapping for every user nor store credentials elsewhere + EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', server_internal); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM + USING HINT = 'Please clean the remaining objects"'; + END; + END IF; + + -- TODO: Drop existing options in server ? + + -- Add new options + FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p + LOOP + IF NOT EXISTS ( + -- TODO: Use pg_options_to_table + WITH a AS ( + SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal + ) SELECT * from a where options = row.key) + THEN + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value); + ELSE + EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value); + END IF; + END LOOP; + + -- TODO: Drop existing user mappings ? + + -- Update user mapping settings + FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o + LOOP + IF NOT EXISTS ( + -- TODO: Use pg_options_to_table + WITH a AS ( + SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = 'public' + ) SELECT * from a where options = option.key) + THEN + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', server_internal, option.key, option.value); + ELSE + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', server_internal, option.key, option.value); + END IF; + END LOOP; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; @@ -184,9 +242,12 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TE RETURNS void AS $$ DECLARE - final_name text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server); BEGIN - EXECUTE @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name := final_name, force := true); + EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); + EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); + EXECUTE FORMAT ('DROP ROLE %I', role_name); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index a258c89..c86f5ef 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -184,17 +184,15 @@ RETURNS void AS $$ DECLARE row record; option record; - fdw_objects_name NAME := fdw_input_name; + fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); BEGIN -- TODO: refactor with original function -- This function tries to be as idempotent as possible, by not creating anything more than once -- (not even using IF NOT EXIST to avoid throwing warnings) IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN CREATE EXTENSION postgres_fdw; - RAISE EXCEPTION 'postgres_fdw extension is not installed' - USING HINT = 'Please install it with "CREATE EXTENSION postgres_fdw"'; + RAISE NOTICE 'Created postgres_fdw EXTENSION'; END IF; - -- Create FDW first if it does not exist IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name) THEN @@ -271,7 +269,7 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false) RETURNS void AS $$ DECLARE - fdw_objects_name NAME := fdw_input_name; + fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); cascade_clause NAME; BEGIN CASE force From e22aaee5f5b6bf39c2573c1b222222822ed190c1 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 16:47:35 +0100 Subject: [PATCH 010/108] Test re-registration of a server with different options --- scripts-available/CDB_FederatedServer.sql | 13 ++++--------- test/CDB_FederatedServer.sql | 19 ++++++++++++++++--- test/CDB_FederatedServer_expect | 6 ++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index d33c7fa..8328c5b 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -167,11 +167,11 @@ $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +-- Registers a new server CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB) RETURNS void AS $$ DECLARE - -- TODO: Check and handle existing servers (if needed) server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); role_name name; @@ -200,14 +200,11 @@ BEGIN USING HINT = 'Please clean the remaining objects"'; END; END IF; - - -- TODO: Drop existing options in server ? - + -- Add new options FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p LOOP IF NOT EXISTS ( - -- TODO: Use pg_options_to_table WITH a AS ( SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal ) SELECT * from a where options = row.key) @@ -217,14 +214,11 @@ BEGIN EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value); END IF; END LOOP; - - -- TODO: Drop existing user mappings ? - + -- Update user mapping settings FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o LOOP IF NOT EXISTS ( - -- TODO: Use pg_options_to_table WITH a AS ( SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = 'public' ) SELECT * from a where options = option.key) @@ -238,6 +232,7 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- Drops a registered server and all the objects associated with it CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT) RETURNS void AS $$ diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index a445e73..58a033b 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -43,8 +43,22 @@ SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server := 'myRemote'); -- Re-register the second server --- SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(); --- SELECT '3.2', cartodb.CDB_PG_Federated_Server_List_Servers(); +SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "other_remote_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); -- Unregister #1 SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote'::text); @@ -52,7 +66,6 @@ SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); -- Unregister a server that doesn't exist SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist'::text); -SELECT '5.2', cartodb.CDB_Federated_Server_List_Servers(); -- Unregister #2 SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index 768b271..24c6479 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -4,8 +4,10 @@ 2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) 2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +3.1| +3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) 4.1| -4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) ERROR: Server "doesNotExist" does not exist -5.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) 6.1| From 144c4e722bf9d4dbbdc18d63e4f941662c3fb051 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:26:36 +0100 Subject: [PATCH 011/108] Add federated server list remotes Includes: - CDB_Federated_Server_List_Remote_Schemas - CDB_Federated_Server_List_Remote_Tables --- scripts-available/CDB_FederatedServer.sql | 70 ++++++----- .../CDB_FederatedServer_ListRemote.sql | 114 +++++++++++++++++ .../401-CDB_FederatedServer_ListRemote.sql | 1 + test/CDB_FederatedServer_ListRemote.sql | 117 ++++++++++++++++++ test/CDB_FederatedServer_ListRemote_expect | 13 ++ 5 files changed, 286 insertions(+), 29 deletions(-) create mode 100644 scripts-available/CDB_FederatedServer_ListRemote.sql create mode 120000 scripts-enabled/401-CDB_FederatedServer_ListRemote.sql create mode 100644 test/CDB_FederatedServer_ListRemote.sql create mode 100644 test/CDB_FederatedServer_ListRemote_expect diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 8328c5b..6f0ec59 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -11,20 +11,19 @@ AS $$ $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - -- Produce a valid DB name for servers generated for the Federated Server CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) RETURNS NAME AS $$ DECLARE - object_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); + internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); BEGIN -- We discard anything that would be truncated - IF (char_length(object_name) < 64) THEN - IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = object_name))) THEN + IF (char_length(internal_server_name) < 64) THEN + IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN RAISE EXCEPTION 'Server "%" does not exist', input_name; END IF; - RETURN object_name::name; + RETURN internal_server_name::name; ELSE RAISE EXCEPTION 'Server name is too long to be used as identifier'; END IF; @@ -32,21 +31,20 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(fdw_stored_name NAME) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME) RETURNS TEXT AS $$ - SELECT right(fdw_stored_name, - char_length(fdw_stored_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; + SELECT right(internal_server_name, + char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- Produce a valid name for a schema generated for the Federated Server -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(server_name TEXT, schema_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name TEXT, schema_name TEXT) RETURNS NAME AS $$ DECLARE - server_full_name text := @extschema@.__CDB_FS_Generate_Server_Name(server_name, check_existence := true); - hash_value text := md5(server_full_name::text || '__' || schema_name::text); + hash_value text := md5(internal_server_name::text || '__' || schema_name::text); schema_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value); BEGIN RETURN schema_name::name; @@ -56,12 +54,11 @@ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- Produce a valid name for a role generated for the Federated Server -- This needs to include the current database in its hash to avoid collisions in clusters with more than one database -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(server_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name TEXT) RETURNS NAME AS $$ DECLARE - server_full_name text := @extschema@.__CDB_FS_Generate_Server_Name(server_name, check_existence := true); - hash_value text := md5(current_database()::text || '__' || server_full_name::text); + hash_value text := md5(current_database()::text || '__' || internal_server_name::text); role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value); BEGIN RETURN role_name::name; @@ -70,22 +67,36 @@ $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- Creates (if not exist) a schema to place the objects for a remote schema -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(server_name TEXT, schema_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name TEXT, schema_name TEXT) RETURNS NAME AS $$ DECLARE - schema_name text := @extschema@.__CDB_FS_Generate_Schema_Name(server_name, schema_name); - role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_name); + schema_name text := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); + role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); BEGIN - BEGIN - EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'TODO: This needs a better error handling after reviewing permissions'; - END; + IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN + BEGIN + EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'TODO: This needs a better error handling after reviewing permissions: %', SQLERRM; + END; + END IF; + RETURN schema_name; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- Returns the type of a server by internal name +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(remote_server name) +RETURNS name +AS $$ + SELECT f.fdwname + FROM pg_foreign_server s + JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid + WHERE s.srvname = remote_server; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + -- List registered servers -- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) @@ -109,7 +120,7 @@ BEGIN @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", -- Which driver are we using (postgres_fdw, odbc_fdw...) - f.fdwname::text AS "Driver", + @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver", -- Read options from pg_foreign_server (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", @@ -120,7 +131,6 @@ BEGIN -- Read username from pg_user_mappings (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" FROM pg_foreign_server s - JOIN pg_foreign_data_wrapper f ON f.oid=s.srvfdw LEFT JOIN pg_user_mappings u ON u.srvid = s.oid WHERE s.srvname ILIKE server_name @@ -137,10 +147,13 @@ AS $$ DECLARE user_mapping jsonb := json_build_object( 'user_mapping', - jsonb_build_object('user', input_config->'credentials'->'username', - 'password', input_config->'credentials'->'password') + jsonb_build_object( 'user', input_config->'credentials'->'username', + 'password', input_config->'credentials'->'password') ); BEGIN + IF NOT (input_config ? 'credentials') THEN + RAISE EXCEPTION 'Credentials are mandatory'; + END IF; RETURN (input_config - 'credentials')::jsonb || user_mapping; END $$ @@ -174,7 +187,7 @@ AS $$ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); - role_name name; + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); row record; option record; BEGIN @@ -187,7 +200,6 @@ BEGIN IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN BEGIN EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); - role_name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server); EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); @@ -238,7 +250,7 @@ RETURNS void AS $$ DECLARE server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); diff --git a/scripts-available/CDB_FederatedServer_ListRemote.sql b/scripts-available/CDB_FederatedServer_ListRemote.sql new file mode 100644 index 0000000..283fc0f --- /dev/null +++ b/scripts-available/CDB_FederatedServer_ListRemote.sql @@ -0,0 +1,114 @@ +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name) +RETURNS TABLE(remote_schema name) +AS $$ +DECLARE + -- Import schemata from the information schema + -- + -- "The view schemata contains all schemas in the current database + -- that the current user has access to (by way of being the owner + -- or having some privilege)." + -- See https://www.postgresql.org/docs/11/infoschema-schemata.html + -- + -- "The information schema is defined in the SQL standard and can + -- therefore be expected to be portable and remain stable" + -- See https://www.postgresql.org/docs/11/information-schema.html + inf_schema name := 'information_schema'; + remote_table name := 'schemata'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); +BEGIN + -- Import the foreign schemata if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); + END IF; + + -- Return the result we're interested in + RETURN QUERY EXECUTE format('SELECT schema_name::name AS remote_schema FROM %I.%I', local_schema, remote_table); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name) +RETURNS TABLE(remote_table name) +AS $func$ +DECLARE + -- Import `tables` from the information schema + -- + -- "The view tables contains all tables and views defined in the + -- current database. Only those tables and views are shown that + -- the current user has access to (by way of being the owner or + -- having some privilege)." + -- https://www.postgresql.org/docs/11/infoschema-tables.html + + -- Create local target schema if it does not exists + inf_schema name := 'information_schema'; + remote_table name := 'tables'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); +BEGIN + -- Import the foreign `tables` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = 'tables' + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); + END IF; + + -- Return the result we're interested in + -- Note: in this context, schema names are not to be quoted + RETURN QUERY EXECUTE format($q$SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s'$q$, local_schema, remote_table, remote_schema); +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + +-- +-- List remote schemas in a federated server that the current user has access to. +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(remote_server name) +RETURNS TABLE(remote_schema name) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List remote tables in a federated server that the current user has access to. +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(remote_server name, remote_schema name) +RETURNS TABLE(remote_table name) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql b/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql new file mode 120000 index 0000000..06fa976 --- /dev/null +++ b/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql @@ -0,0 +1 @@ +../scripts-available/CDB_FederatedServer_ListRemote.sql \ No newline at end of file diff --git a/test/CDB_FederatedServer_ListRemote.sql b/test/CDB_FederatedServer_ListRemote.sql new file mode 100644 index 0000000..532d3bf --- /dev/null +++ b/test/CDB_FederatedServer_ListRemote.sql @@ -0,0 +1,117 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +SET SESSION AUTHORIZATION postgres; +CREATE EXTENSION postgres_fdw; +CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback2'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +-- =================================================================== +-- create objects used through FDW loopback server +-- =================================================================== +\c cdb_fs_tester cdb_fs_tester + +CREATE TYPE user_enum AS ENUM ('foo', 'bar', 'buz'); +CREATE SCHEMA "S 1"; +CREATE TABLE "S 1"."T 1" ( + "C 1" int NOT NULL, + c2 int NOT NULL, + c3 text, + c4 timestamptz, + c5 timestamp, + c6 varchar(10), + c7 char(10), + c8 user_enum, + CONSTRAINT t1_pkey PRIMARY KEY ("C 1") +); +CREATE TABLE "S 1"."T 2" ( + c1 int NOT NULL, + c2 text, + CONSTRAINT t2_pkey PRIMARY KEY (c1) +); +CREATE TABLE "S 1"."T 3" ( + c1 int NOT NULL, + c2 int NOT NULL, + c3 text, + CONSTRAINT t3_pkey PRIMARY KEY (c1) +); +CREATE TABLE "S 1"."T 4" ( + c1 int NOT NULL, + c2 int NOT NULL, + c3 text, + CONSTRAINT t4_pkey PRIMARY KEY (c1) +); + +-- Disable autovacuum for these tables to avoid unexpected effects of that +ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 3" SET (autovacuum_enabled = 'false'); +ALTER TABLE "S 1"."T 4" SET (autovacuum_enabled = 'false'); + +\c contrib_regression postgres +SET client_min_messages TO notice; +\set VERBOSITY terse +\set QUIET off + + +-- =================================================================== +-- Test the listing functions +-- =================================================================== +\echo 'Test listing of remote schemas (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(remote_server => 'loopback') + WHERE remote_schema NOT LIKE 'pg_%' -- Exclude toast and temp schemas + ORDER BY remote_schema; + +\echo 'Test listing of remote tables (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(remote_server => 'loopback', remote_schema => 'S 1') + ORDER BY remote_table; + +-- =================================================================== +-- Cleanup +-- =================================================================== +\set QUIET on + +\c cdb_fs_tester cdb_fs_tester +DROP TABLE "S 1". "T 1"; +DROP TABLE "S 1". "T 2"; +DROP TABLE "S 1". "T 3"; +DROP TABLE "S 1". "T 4"; + +DROP SCHEMA "S 1"; +DROP TYPE user_enum; + +\c contrib_regression postgres +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse + +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); +SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); +DROP DATABASE cdb_fs_tester; +DROP ROLE cdb_fs_tester; +\set QUIET off diff --git a/test/CDB_FederatedServer_ListRemote_expect b/test/CDB_FederatedServer_ListRemote_expect new file mode 100644 index 0000000..19c9022 --- /dev/null +++ b/test/CDB_FederatedServer_ListRemote_expect @@ -0,0 +1,13 @@ +C1| +C2| +Test listing of remote schemas (sunny day) +S 1 +information_schema +public +Test listing of remote tables (sunny day) +T 1 +T 2 +T 3 +T 4 +D1| +D2| From f7ab3d28a863ea15657d0b69dbbea6b172df3ca2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:35:02 +0100 Subject: [PATCH 012/108] __CDB_FS_List_Foreign_Schemas_PG: Handle order and ignore pg_% schemas automatically --- scripts-available/CDB_FederatedServer_ListRemote.sql | 12 +++++++++--- test/CDB_FederatedServer_ListRemote.sql | 7 ++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_FederatedServer_ListRemote.sql b/scripts-available/CDB_FederatedServer_ListRemote.sql index 283fc0f..e445cf2 100644 --- a/scripts-available/CDB_FederatedServer_ListRemote.sql +++ b/scripts-available/CDB_FederatedServer_ListRemote.sql @@ -28,8 +28,12 @@ BEGIN EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); END IF; - -- Return the result we're interested in - RETURN QUERY EXECUTE format('SELECT schema_name::name AS remote_schema FROM %I.%I', local_schema, remote_table); + -- Return the result we're interested in. Exclude toast and temp schemas + RETURN QUERY EXECUTE format(' + SELECT schema_name::name AS remote_schema FROM %I.%I + WHERE schema_name NOT LIKE %s + ORDER BY remote_schema + ', local_schema, remote_table, '''pg_%'''); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; @@ -63,7 +67,9 @@ BEGIN -- Return the result we're interested in -- Note: in this context, schema names are not to be quoted - RETURN QUERY EXECUTE format($q$SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s'$q$, local_schema, remote_table, remote_schema); + RETURN QUERY EXECUTE format($q$ + SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name + $q$, local_schema, remote_table, remote_schema); END $func$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServer_ListRemote.sql b/test/CDB_FederatedServer_ListRemote.sql index 532d3bf..ff2ac36 100644 --- a/test/CDB_FederatedServer_ListRemote.sql +++ b/test/CDB_FederatedServer_ListRemote.sql @@ -83,13 +83,10 @@ SET client_min_messages TO notice; -- Test the listing functions -- =================================================================== \echo 'Test listing of remote schemas (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(remote_server => 'loopback') - WHERE remote_schema NOT LIKE 'pg_%' -- Exclude toast and temp schemas - ORDER BY remote_schema; +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(remote_server => 'loopback'); \echo 'Test listing of remote tables (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(remote_server => 'loopback', remote_schema => 'S 1') - ORDER BY remote_table; +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(remote_server => 'loopback', remote_schema => 'S 1'); -- =================================================================== -- Cleanup From d85faa8f8508ef51285044613e28d3610be4630e Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:39:23 +0100 Subject: [PATCH 013/108] Now trully fix installcheck target --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0f902aa..b7d41f3 100644 --- a/Makefile +++ b/Makefile @@ -220,6 +220,6 @@ regress: legacy_tests PGPORT=$(PGPORT) \ $(PGREGRESS) --inputdir=./ --bindir='$(PGBINDIR)' --dbname=$(PGREGRESSDATABASE) $(REGRESS) -installcheck: test_extension_new test_organization regress - $(MAKE) -f ... -C regress +installcheck: test_extension_new test_organization + $(MAKE) -C . regress From 664468b1e594de83105d3b0b525f3c7005816c99 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:50:57 +0100 Subject: [PATCH 014/108] Rename CDB_FederatedServerListRemote files --- ...edServer_ListRemote.sql => CDB_FederatedServerListRemote.sql} | 0 scripts-enabled/401-CDB_FederatedServer_ListRemote.sql | 1 - scripts-enabled/402_CDB_FederatedServerListRemote.sql | 1 + ...edServer_ListRemote.sql => CDB_FederatedServerListRemote.sql} | 0 ...er_ListRemote_expect => CDB_FederatedServerListRemote_expect} | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename scripts-available/{CDB_FederatedServer_ListRemote.sql => CDB_FederatedServerListRemote.sql} (100%) delete mode 120000 scripts-enabled/401-CDB_FederatedServer_ListRemote.sql create mode 120000 scripts-enabled/402_CDB_FederatedServerListRemote.sql rename test/{CDB_FederatedServer_ListRemote.sql => CDB_FederatedServerListRemote.sql} (100%) rename test/{CDB_FederatedServer_ListRemote_expect => CDB_FederatedServerListRemote_expect} (100%) diff --git a/scripts-available/CDB_FederatedServer_ListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql similarity index 100% rename from scripts-available/CDB_FederatedServer_ListRemote.sql rename to scripts-available/CDB_FederatedServerListRemote.sql diff --git a/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql b/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql deleted file mode 120000 index 06fa976..0000000 --- a/scripts-enabled/401-CDB_FederatedServer_ListRemote.sql +++ /dev/null @@ -1 +0,0 @@ -../scripts-available/CDB_FederatedServer_ListRemote.sql \ No newline at end of file diff --git a/scripts-enabled/402_CDB_FederatedServerListRemote.sql b/scripts-enabled/402_CDB_FederatedServerListRemote.sql new file mode 120000 index 0000000..08cfcc6 --- /dev/null +++ b/scripts-enabled/402_CDB_FederatedServerListRemote.sql @@ -0,0 +1 @@ +CDB_FederatedServerListRemote.sql \ No newline at end of file diff --git a/test/CDB_FederatedServer_ListRemote.sql b/test/CDB_FederatedServerListRemote.sql similarity index 100% rename from test/CDB_FederatedServer_ListRemote.sql rename to test/CDB_FederatedServerListRemote.sql diff --git a/test/CDB_FederatedServer_ListRemote_expect b/test/CDB_FederatedServerListRemote_expect similarity index 100% rename from test/CDB_FederatedServer_ListRemote_expect rename to test/CDB_FederatedServerListRemote_expect From b76b754811ec45b0184488090a29f0086c51f992 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:53:05 +0100 Subject: [PATCH 015/108] CDB_FederatedServerTables skeleton --- scripts-available/CDB_FederatedServerTables.sql | 0 scripts-enabled/401_CDB_FederatedServerTables.sql | 1 + test/CDB_FederatedServerTables.sql | 0 test/CDB_FederatedServerTables_expect | 0 4 files changed, 1 insertion(+) create mode 100644 scripts-available/CDB_FederatedServerTables.sql create mode 120000 scripts-enabled/401_CDB_FederatedServerTables.sql create mode 100644 test/CDB_FederatedServerTables.sql create mode 100644 test/CDB_FederatedServerTables_expect diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql new file mode 100644 index 0000000..e69de29 diff --git a/scripts-enabled/401_CDB_FederatedServerTables.sql b/scripts-enabled/401_CDB_FederatedServerTables.sql new file mode 120000 index 0000000..87da93b --- /dev/null +++ b/scripts-enabled/401_CDB_FederatedServerTables.sql @@ -0,0 +1 @@ +CDB_FederatedServerTables.sql \ No newline at end of file diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql new file mode 100644 index 0000000..e69de29 diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect new file mode 100644 index 0000000..e69de29 From 77868a541d088f93932d25cdf87efc530f350533 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 29 Oct 2019 18:55:18 +0100 Subject: [PATCH 016/108] Fix symlinks :D --- scripts-enabled/401_CDB_FederatedServerTables.sql | 2 +- scripts-enabled/402_CDB_FederatedServerListRemote.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-enabled/401_CDB_FederatedServerTables.sql b/scripts-enabled/401_CDB_FederatedServerTables.sql index 87da93b..e835b08 120000 --- a/scripts-enabled/401_CDB_FederatedServerTables.sql +++ b/scripts-enabled/401_CDB_FederatedServerTables.sql @@ -1 +1 @@ -CDB_FederatedServerTables.sql \ No newline at end of file +../scripts-available/CDB_FederatedServerTables.sql \ No newline at end of file diff --git a/scripts-enabled/402_CDB_FederatedServerListRemote.sql b/scripts-enabled/402_CDB_FederatedServerListRemote.sql index 08cfcc6..e6f0b52 120000 --- a/scripts-enabled/402_CDB_FederatedServerListRemote.sql +++ b/scripts-enabled/402_CDB_FederatedServerListRemote.sql @@ -1 +1 @@ -CDB_FederatedServerListRemote.sql \ No newline at end of file +../scripts-available/CDB_FederatedServerListRemote.sql \ No newline at end of file From a26b96347a7109e859e26ca7eeaa2a5e97b3dd57 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 11:40:35 +0100 Subject: [PATCH 017/108] Initial implementation of CDB_Federated_Table_Register --- scripts-available/CDB_FederatedServer.sql | 2 +- .../CDB_FederatedServerListRemote.sql | 11 +- .../CDB_FederatedServerTables.sql | 173 ++++++++++++++++++ test/CDB_FederatedServerListRemote.sql | 5 +- test/CDB_FederatedServerTables.sql | 82 +++++++++ test/CDB_FederatedServerTables_expect | 10 + 6 files changed, 275 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 6f0ec59..2d6375b 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -209,7 +209,7 @@ BEGIN EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', server_internal); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM - USING HINT = 'Please clean the remaining objects"'; + USING HINT = 'Please clean the left over objects'; END; END IF; diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index e445cf2..6e66dbc 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -18,6 +18,7 @@ DECLARE inf_schema name := 'information_schema'; remote_table name := 'schemata'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); + role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN -- Import the foreign schemata if not done IF NOT EXISTS ( @@ -82,18 +83,18 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List remote schemas in a federated server that the current user has access to. -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(remote_server name) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT) RETURNS TABLE(remote_schema name) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type WHEN 'postgres_fdw' THEN RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal); ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; END CASE; END $$ @@ -102,11 +103,11 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List remote tables in a federated server that the current user has access to. -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(remote_server name, remote_schema name) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT) RETURNS TABLE(remote_table name) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index e69de29..8f32b59 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -0,0 +1,173 @@ +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid IN (SELECT oid FROM pg_type + WHERE typname IN + ('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8')); + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid = 'geometry'::regtype; + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS) +RETURNS SETOF NAME +AS $$ + SELECT + a.attname as "colname" + FROM pg_catalog.pg_attribute a + WHERE + a.attnum > 0 + AND NOT a.attisdropped + AND a.attrelid = ( + SELECT c.oid + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = input_table::oid + ) + ORDER BY a.attnum; +$$ LANGUAGE SQL; + + + +-- +-- Set up a Federated Table +-- +-- Precondition: the federated server has to be set up via +-- CDB_SetUp_PG_Federated_Server +-- +-- Postcondition: it generates a view in the schema of the user that +-- can be used through SQL and Maps API's. +-- +-- E.g: +-- SELECT cartodb.CDB_SetUp_PG_Federated_Table( +-- 'amazon', -- mandatory, name of the federated server +-- 'my_remote_schema', -- mandatory, schema name +-- 'my_remote_table', -- mandatory, table name +-- 'id', -- mandatory, name of the id column +-- 'geom', -- optional, name of the geom column, preferably in 4326 +-- 'webmercator' -- optional, should be in 3857 if present +-- ); +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( + server TEXT, + remote_schema TEXT, + remote_table TEXT, + id_column TEXT, + geom_column TEXT DEFAULT NULL, + webmercator_column TEXT DEFAULT NULL, + local_name NAME DEFAULT NULL +) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + src_table REGCLASS; + + rest_of_cols TEXT[]; + geom_expression TEXT; + webmercator_expression TEXT; + carto_columns_expression TEXT[]; +BEGIN + -- Use geom_column as default for webmercator_column + IF webmercator_column IS NULL THEN + webmercator_column := geom_column; + END IF; + + IF local_name IS NULL THEN + local_name := remote_table; + END IF; + + -- Import the foreign table + EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', remote_schema, remote_table, server_internal, local_schema); + src_table := format('%I.%I', local_schema, remote_table); + + --- Grant SELECT to fdw role (TODO: Re-enable if needed) + --- EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); + + -- Check id_column is numeric + IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN + RAISE EXCEPTION 'non integer id_column "%"', id_column; + END IF; + + -- Check if the geom and mercator columns have a geometry type (if provided) + IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN + RAISE EXCEPTION 'non geometry column "%"', geom_column; + END IF; + IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN + RAISE EXCEPTION 'non geometry column "%"', webmercator_column; + END IF; + + -- Get a list of columns excluding the id, geom and the_geom_webmercator + SELECT ARRAY( + SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c + WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL) + ) INTO rest_of_cols; + + IF geom_column IS NULL + THEN + geom_expression := 'NULL AS the_geom'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326 + THEN + geom_expression := format('t.%I AS the_geom', geom_column); + ELSE + -- It needs an ST_Transform to 4326 + geom_expression := format('@postgisschema@.ST_Transform(t.%I, 4326) AS the_geom', geom_column); + END IF; + + IF webmercator_column IS NULL + THEN + webmercator_expression := 'NULL AS the_geom_webmercator'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857 + THEN + webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column); + ELSE + -- It needs an ST_Transform to 3857 + webmercator_expression := format('@postgisschema@.ST_Transform(t.%I, 3857) AS the_geom_webmercator', webmercator_column); + END IF; + + -- CARTO columns expressions + carto_columns_expression := ARRAY[ + format('t.%1$I AS cartodb_id', id_column), + geom_expression, + webmercator_expression + ]; + + -- Create a view with homogeneous CDB fields + BEGIN + EXECUTE format( + 'CREATE OR REPLACE VIEW %1$I AS + SELECT %2s + FROM %3$s t', + local_name, + array_to_string(carto_columns_expression || rest_of_cols, ','), + src_table + ); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; + END; + + -- TODO: Handle this Grant perms to the view + -- EXECUTE format('GRANT SELECT ON %I TO %s', table_name, fdw_objects_name); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index ff2ac36..1e0289d 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -83,10 +83,10 @@ SET client_min_messages TO notice; -- Test the listing functions -- =================================================================== \echo 'Test listing of remote schemas (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(remote_server => 'loopback'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \echo 'Test listing of remote tables (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(remote_server => 'loopback', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -- =================================================================== -- Cleanup @@ -111,4 +111,5 @@ SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text) SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); DROP DATABASE cdb_fs_tester; DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index e69de29..a336a20 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -0,0 +1,82 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +SET SESSION AUTHORIZATION postgres; +CREATE EXTENSION postgres_fdw; +CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + + +-- =================================================================== +-- create objects used through FDW loopback server +-- =================================================================== + +\c cdb_fs_tester postgres + +CREATE EXTENSION postgis; + +\c cdb_fs_tester cdb_fs_tester + +CREATE SCHEMA remote_schema; +CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326)); + +INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry); +INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry); + +CREATE TABLE remote_schema.remote_geom2(id bigint, another_field text, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); + +INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)'); + +CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text); +INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes'); + + +-- =================================================================== +-- Test the listing functions +-- =================================================================== + +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse +\set QUIET off + +SELECT 'R1', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +SELECT 'V1', pg_get_viewdef('remote_geom'); +SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; + + +-- =================================================================== +-- Cleanup +-- =================================================================== + +\set QUIET on +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse + +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); +DROP DATABASE cdb_fs_tester; +DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index e69de29..75299aa 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -0,0 +1,10 @@ +C1| +R1| +V1| SELECT t.id AS cartodb_id, + t.geom AS the_geom, + st_transform(t.geom, 3857) AS the_geom_webmercator, + t.another_field + FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom t; +S1|1|POINT(1 1)|patata +S1|2|POINT(2 2)|patata2 +D1| From 7bb0b93f8bdc001688417ec08f6be7855e1579c0 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 11:47:45 +0100 Subject: [PATCH 018/108] Test renaming --- test/CDB_FederatedServerTables.sql | 13 +++++++++++++ test/CDB_FederatedServerTables_expect | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index a336a20..efd1074 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -65,6 +65,19 @@ SELECT 'R1', cartodb.CDB_Federated_Table_Register( SELECT 'V1', pg_get_viewdef('remote_geom'); SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; +SELECT 'R2', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom2', + id_column => 'id', + geom_column => 'geom', + webmercator_column => 'geom_mercator', + local_name => 'myFullTable' + ); + +SELECT 'V2', pg_get_viewdef('"myFullTable"'); +SELECT 'S2', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 75299aa..78b44d1 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -7,4 +7,11 @@ V1| SELECT t.id AS cartodb_id, FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom t; S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 +R2| +V2| SELECT t.id AS cartodb_id, + t.geom AS the_geom, + t.geom_mercator AS the_geom_webmercator, + t.another_field + FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom2 t; +S2|3|POINT(3 3)|patata D1| From d5f84784f07c14085d906459c4c8be366eb15cb8 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 11:55:02 +0100 Subject: [PATCH 019/108] Add CDB_Federated_Table_Unregister --- .../CDB_FederatedServerTables.sql | 19 +++++++++++++++++++ test/CDB_FederatedServerTables.sql | 9 +++++++++ test/CDB_FederatedServerTables_expect | 2 ++ 3 files changed, 30 insertions(+) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 8f32b59..913fe32 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -171,3 +171,22 @@ BEGIN END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister( + server TEXT, + remote_schema TEXT, + remote_table TEXT +) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table); + + +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index efd1074..42afbb0 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -78,6 +78,15 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( SELECT 'V2', pg_get_viewdef('"myFullTable"'); SELECT 'S2', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; +SELECT 'U1', CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); +-- Selecting from the created view should fail now +SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; + + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 78b44d1..a8fdf78 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -14,4 +14,6 @@ V2| SELECT t.id AS cartodb_id, t.another_field FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom2 t; S2|3|POINT(3 3)|patata +U1| +ERROR: relation "remote_geom" does not exist at character 71 D1| From b83923580194021fe7f99f74db52d8aadd0deead Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 12:19:30 +0100 Subject: [PATCH 020/108] Add CDB_Federated_Server_List_Registered_Tables --- .../CDB_FederatedServerTables.sql | 35 +++++++++++++++++-- test/CDB_FederatedServerTables.sql | 15 +++++++- test/CDB_FederatedServerTables_expect | 4 +++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 913fe32..3b02ac9 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -185,8 +185,39 @@ DECLARE local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); BEGIN EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table); - - +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Registered_Tables( + server TEXT, + remote_schema TEXT + ) +RETURNS TABLE(remote_name TEXT, local_name TEXT) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + RETURN QUERY SELECT + source_table::text as remote_table, + format('%I.%I', dependent_schema, dependent_view)::text as local_view + FROM + ( + SELECT DISTINCT + dependent_ns.nspname as dependent_schema, + dependent_view.relname as dependent_view, + source_table.relname as source_table + FROM pg_depend + JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid + JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid + JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid + JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace + JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace + WHERE + source_ns.nspname = local_schema + ORDER BY 1,2 + ) _aux; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 42afbb0..1072f47 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -65,6 +65,11 @@ SELECT 'R1', cartodb.CDB_Federated_Table_Register( SELECT 'V1', pg_get_viewdef('remote_geom'); SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; +Select 'list_remotes1', CDB_Federated_Server_List_Registered_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); + SELECT 'R2', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -78,6 +83,11 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( SELECT 'V2', pg_get_viewdef('"myFullTable"'); SELECT 'S2', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; +Select 'list_remotes2', CDB_Federated_Server_List_Registered_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); + SELECT 'U1', CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', @@ -86,7 +96,10 @@ SELECT 'U1', CDB_Federated_Table_Unregister( -- Selecting from the created view should fail now SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; - +Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( + server => 'loopback', + remote_schema => 'remote_schema' +); -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index a8fdf78..0d0f175 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -7,6 +7,7 @@ V1| SELECT t.id AS cartodb_id, FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom t; S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 +list_remotes1|(remote_geom,public.remote_geom) R2| V2| SELECT t.id AS cartodb_id, t.geom AS the_geom, @@ -14,6 +15,9 @@ V2| SELECT t.id AS cartodb_id, t.another_field FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom2 t; S2|3|POINT(3 3)|patata +list_remotes2|(remote_geom2,"public.""myFullTable""") +list_remotes2|(remote_geom,public.remote_geom) U1| ERROR: relation "remote_geom" does not exist at character 71 +list_remotes3|(remote_geom2,"public.""myFullTable""") D1| From cc5fb37abbc29e801d5f3274ad2eca880acb40ad Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 13:16:31 +0100 Subject: [PATCH 021/108] CDB_FederatedServer: Improve comments --- scripts-available/CDB_FederatedServer.sql | 143 +++++++++++++++------- 1 file changed, 97 insertions(+), 46 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 2d6375b..324359f 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -1,9 +1,14 @@ +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- +-- -- This function is just a placement to store and use the pattern for --- foreign server names +-- foreign object names -- Servers: cdb_fs_$(server_name) -- Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) -- Owner role: cdb_fs_$(md5sum(current_database() || server_name) +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() RETURNS TEXT AS $$ @@ -11,7 +16,10 @@ AS $$ $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +-- -- Produce a valid DB name for servers generated for the Federated Server +-- If check_existence is true, it'll throw if the server doesn't exists +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) RETURNS NAME AS $$ @@ -31,6 +39,10 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +-- +-- Given the internal name for a remote server, it returns the name used by the user +-- Reverses __CDB_FS_Generate_Server_Name +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME) RETURNS TEXT AS $$ @@ -39,7 +51,9 @@ AS $$ $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +-- -- Produce a valid name for a schema generated for the Federated Server +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name TEXT, schema_name TEXT) RETURNS NAME AS $$ @@ -52,8 +66,10 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +-- -- Produce a valid name for a role generated for the Federated Server -- This needs to include the current database in its hash to avoid collisions in clusters with more than one database +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name TEXT) RETURNS NAME AS $$ @@ -66,7 +82,10 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +-- -- Creates (if not exist) a schema to place the objects for a remote schema +-- The schema is with the same AUTHORIZATION as the server +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name TEXT, schema_name TEXT) RETURNS NAME AS $$ @@ -86,7 +105,10 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- -- Returns the type of a server by internal name +-- Currently all of them should be postgres_fdw +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(remote_server name) RETURNS name AS $$ @@ -97,50 +119,9 @@ AS $$ $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; - --- List registered servers --- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) --- TODO: Handle multiple user mappings in the same server -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') -RETURNS TABLE ( - name text, - driver text, - host text, - port text, - dbname text, - readmode text, - username text -) -AS $$ -DECLARE - server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server); -BEGIN - RETURN QUERY SELECT - -- Name as shown to the user - @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", - - -- Which driver are we using (postgres_fdw, odbc_fdw...) - @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver", - - -- Read options from pg_foreign_server - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", - (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", - CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", - - -- Read username from pg_user_mappings - (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" - FROM pg_foreign_server s - LEFT JOIN pg_user_mappings u - ON u.srvid = s.oid - WHERE s.srvname ILIKE server_name - ORDER BY 1; -END -$$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; - --- Take a config jsonb and transform it to an input suitable for --- _CDB_SetUp_User_PG_FDW_Server +-- +-- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB) RETURNS jsonb AS $$ @@ -180,7 +161,32 @@ $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; --- Registers a new server +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + + +-- +-- Registers a new PG server +-- +-- Example config: '{ +-- "server": { +-- "dbname": "fdw_target", +-- "host": "localhost", +-- "port": 5432, +-- "extensions": "postgis", +-- "updatable": "false", +-- "use_remote_estimate": "true", +-- "fetch_size": "1000" +-- }, +-- "credentials": { +-- "username": "fdw_user", +-- "password": "foobarino" +-- } +-- }' +-- +-- The configuration from __CDB_FS_add_default_options will be appended +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB) RETURNS void AS $$ @@ -244,7 +250,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- -- Drops a registered server and all the objects associated with it +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT) RETURNS void AS $$ @@ -258,3 +266,46 @@ BEGIN END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- List registered servers +-- +-- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) +-- TODO: Handle multiple user mappings in the same server +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') +RETURNS TABLE ( + name text, + driver text, + host text, + port text, + dbname text, + readmode text, + username text +) +AS $$ +DECLARE + server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server); +BEGIN + RETURN QUERY SELECT + -- Name as shown to the user + @extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name", + + -- Which driver are we using (postgres_fdw, odbc_fdw...) + @extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver", + + -- Read options from pg_foreign_server + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port", + (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", + CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", + + -- Read username from pg_user_mappings + (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" + FROM pg_foreign_server s + LEFT JOIN pg_user_mappings u + ON u.srvid = s.oid + WHERE s.srvname ILIKE server_name + ORDER BY 1; +END +$$ +LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; From cf6b00dfd8c72b171443720a348e96a09a25bd7c Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 13:20:41 +0100 Subject: [PATCH 022/108] CDB_FederatedServerListRemote: Improve comments --- scripts-available/CDB_FederatedServerListRemote.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 6e66dbc..bc2306c 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -1,6 +1,10 @@ -------------------------------------------------------------------------------- -- Private functions -------------------------------------------------------------------------------- + +-- +-- List the schemas of a remote PG server +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name) RETURNS TABLE(remote_schema name) AS $$ @@ -20,7 +24,7 @@ DECLARE local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN - -- Import the foreign schemata if not done + -- Import the foreign schemata table IF NOT EXISTS ( SELECT * FROM pg_class WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) @@ -39,7 +43,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - +-- +-- List the tables from a remote PG schema +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name) RETURNS TABLE(remote_table name) AS $func$ From f20a00716085fe3d92b98312a52ad6219a9aee40 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 13:27:44 +0100 Subject: [PATCH 023/108] CDB_FederatedServerTables: Some comments --- .../CDB_FederatedServerTables.sql | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 3b02ac9..30750ab 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -1,3 +1,10 @@ +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +-- +-- Checks if a column is of integer type +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME) RETURNS boolean AS $$ @@ -13,7 +20,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - +-- +-- Checks if a column is of geometry type +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME) RETURNS boolean AS $$ @@ -21,13 +30,15 @@ BEGIN PERFORM atttypid FROM pg_catalog.pg_attribute WHERE attrelid = input_table AND attname = colname - AND atttypid = 'geometry'::regtype; + AND atttypid = '@postgisschema@.geometry'::regtype; RETURN FOUND; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - +-- +-- Returns the name of all the columns from a table +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS) RETURNS SETOF NAME AS $$ @@ -47,9 +58,12 @@ AS $$ $$ LANGUAGE SQL; +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- -- --- Set up a Federated Table +-- Sets up a Federated Table -- -- Precondition: the federated server has to be set up via -- CDB_SetUp_PG_Federated_Server @@ -65,6 +79,7 @@ $$ LANGUAGE SQL; -- 'id', -- mandatory, name of the id column -- 'geom', -- optional, name of the geom column, preferably in 4326 -- 'webmercator' -- optional, should be in 3857 if present +-- 'local_name' -- optional, name of the local view (uses the remote_name if not declared) -- ); -- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( @@ -172,7 +187,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - +-- +-- Unregisters a remote table. Any dependent object will be dropped +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister( server TEXT, remote_schema TEXT, @@ -189,6 +206,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- List all registered tables in a server + schema +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Registered_Tables( server TEXT, remote_schema TEXT From 28376ffecb0aa61f304b340356aa1cdf219efba8 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 13:53:56 +0100 Subject: [PATCH 024/108] CDB_Federated_Table_Register works to refresh a table --- scripts-available/CDB_FederatedServerTables.sql | 7 ++++++- test/CDB_FederatedServerTables.sql | 16 ++++++++++++++++ test/CDB_FederatedServerTables_expect | 5 ++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 30750ab..bd93445 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -66,10 +66,11 @@ $$ LANGUAGE SQL; -- Sets up a Federated Table -- -- Precondition: the federated server has to be set up via --- CDB_SetUp_PG_Federated_Server +-- CDB_Federated_Server_Register_PG -- -- Postcondition: it generates a view in the schema of the user that -- can be used through SQL and Maps API's. +-- If the table was already exported, it will be dropped and re-imported -- -- E.g: -- SELECT cartodb.CDB_SetUp_PG_Federated_Table( @@ -113,6 +114,10 @@ BEGIN END IF; -- Import the foreign table + -- Drop the old view / table if there was one + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN + EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table); + END IF; EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', remote_schema, remote_table, server_internal, local_schema); src_table := format('%I.%I', local_schema, remote_table); diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 1072f47..11986f5 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -88,6 +88,22 @@ Select 'list_remotes2', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); +-- Reregistering R2 with different parameters should work +SELECT 'R3', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom2', + id_column => 'id', + geom_column => 'geom', + local_name => 'different_name' + ); + +-- The old view should dissapear +SELECT 'S3_old', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; +-- And the new appear +SELECT 'S3_new', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM different_name; + +-- Deregistering the first table SELECT 'U1', CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 0d0f175..3badec2 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -17,7 +17,10 @@ V2| SELECT t.id AS cartodb_id, S2|3|POINT(3 3)|patata list_remotes2|(remote_geom2,"public.""myFullTable""") list_remotes2|(remote_geom,public.remote_geom) +R3| +ERROR: relation "myFullTable" does not exist at character 82 +S3_new|3|POINT(333958.472379821 334111.17140196)|patata U1| ERROR: relation "remote_geom" does not exist at character 71 -list_remotes3|(remote_geom2,"public.""myFullTable""") +list_remotes3|(remote_geom2,public.different_name) D1| From f75756138a7fa30994a95cb50ecdae311be622a6 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 30 Oct 2019 15:04:01 +0100 Subject: [PATCH 025/108] CDB_FederatedServerTables: Avoid decimal mismatch --- test/CDB_FederatedServerTables.sql | 6 +++--- test/CDB_FederatedServerTables_expect | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 11986f5..460252d 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -81,7 +81,7 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( ); SELECT 'V2', pg_get_viewdef('"myFullTable"'); -SELECT 'S2', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; +SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTable"; Select 'list_remotes2', CDB_Federated_Server_List_Registered_Tables( server => 'loopback', @@ -99,9 +99,9 @@ SELECT 'R3', cartodb.CDB_Federated_Table_Register( ); -- The old view should dissapear -SELECT 'S3_old', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM "myFullTable"; +SELECT 'S3_old', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTable"; -- And the new appear -SELECT 'S3_new', cartodb_id, ST_AsText(the_geom_webmercator), another_field FROM different_name; +SELECT 'S3_new', cartodb_id, ST_AsText(the_geom), another_field FROM different_name; -- Deregistering the first table SELECT 'U1', CDB_Federated_Table_Unregister( diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 3badec2..21229d3 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -18,8 +18,8 @@ S2|3|POINT(3 3)|patata list_remotes2|(remote_geom2,"public.""myFullTable""") list_remotes2|(remote_geom,public.remote_geom) R3| -ERROR: relation "myFullTable" does not exist at character 82 -S3_new|3|POINT(333958.472379821 334111.17140196)|patata +ERROR: relation "myFullTable" does not exist at character 70 +S3_new|3|POINT(3 3)|patata U1| ERROR: relation "remote_geom" does not exist at character 71 list_remotes3|(remote_geom2,public.different_name) From eea01988649b14695b95b05f0e42c32cef69a3db Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 31 Oct 2019 15:29:37 +0100 Subject: [PATCH 026/108] Add CDB_Federated_Server_List_Remote_Columns --- .../CDB_FederatedServerListRemote.sql | 62 ++++++++++++++++++- test/CDB_FederatedServerListRemote.sql | 3 + test/CDB_FederatedServerListRemote_expect | 9 +++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index bc2306c..84b4f81 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -67,7 +67,7 @@ BEGIN IF NOT EXISTS ( SELECT * FROM pg_class WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = 'tables' + AND relname = remote_table ) THEN EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); END IF; @@ -82,6 +82,46 @@ $func$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- List the columns from a remote PG table +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name, remote_table name) +RETURNS TABLE(column_name name) +AS $func$ +DECLARE + -- Import `columns` from the information schema + -- + -- "The view columns contains information about all table columns (or view columns) + -- in the database. System columns (oid, etc.) are not included. Only those columns + -- are shown that the current user has access to (by way of being the owner or having some privilege)." + -- https://www.postgresql.org/docs/11/infoschema-columns.html + + -- Create local target schema if it does not exists + inf_schema name := 'information_schema'; + remote_col_table name := 'columns'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); + sql_q text; +BEGIN + -- Import the foreign `columns` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_col_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_col_table, server_internal, local_schema); + END IF; + + -- Return the result we're interested in + -- Note: in this context, remote schema and remote table names are not to be quoted + RETURN QUERY EXECUTE format($q$ + SELECT column_name::name FROM %I.%I + WHERE table_schema = '%s' AND table_name = '%s' + ORDER BY column_name$q$, + local_schema, remote_col_table, remote_schema, remote_table); +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + -------------------------------------------------------------------------------- -- Public functions -------------------------------------------------------------------------------- @@ -125,3 +165,23 @@ BEGIN END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( + server TEXT, + remote_schema TEXT, + remote_table TEXT) +RETURNS TABLE(column_name name) +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); +BEGIN + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema, remote_table); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + END CASE; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index 1e0289d..07b0344 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -88,6 +88,9 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopba \echo 'Test listing of remote tables (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\echo 'Test listing of remote columns (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); + -- =================================================================== -- Cleanup -- =================================================================== diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index 19c9022..f36517b 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -9,5 +9,14 @@ T 1 T 2 T 3 T 4 +Test listing of remote columns (sunny day) +C 1 +c2 +c3 +c4 +c5 +c6 +c7 +c8 D1| D2| From c740cfa499c48d5f25de1f1d71e7f8f4a23f00ca Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Nov 2019 15:18:46 +0100 Subject: [PATCH 027/108] CDB_Federated_Server_List_Remote_Columns: Return also type name --- .../CDB_FederatedServerListRemote.sql | 66 +++++++++++++++++-- test/CDB_FederatedServerListRemote.sql | 50 +++++++++++++- test/CDB_FederatedServerListRemote_expect | 27 +++++--- 3 files changed, 127 insertions(+), 16 deletions(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 84b4f81..7cc3fb4 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -86,7 +86,7 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- List the columns from a remote PG table -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name, remote_table name) -RETURNS TABLE(column_name name) +RETURNS TABLE(column_name name, column_type text) AS $func$ DECLARE -- Import `columns` from the information schema @@ -100,7 +100,6 @@ DECLARE inf_schema name := 'information_schema'; remote_col_table name := 'columns'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); - sql_q text; BEGIN -- Import the foreign `columns` if not done IF NOT EXISTS ( @@ -114,14 +113,69 @@ BEGIN -- Return the result we're interested in -- Note: in this context, remote schema and remote table names are not to be quoted RETURN QUERY EXECUTE format($q$ - SELECT column_name::name FROM %I.%I + SELECT + a.column_name::name, COALESCE(b.column_type, a.data_type)::TEXT as column_type + FROM %I.%I a + LEFT JOIN @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s', '%s') b ON a.column_name = b.column_name WHERE table_schema = '%s' AND table_name = '%s' ORDER BY column_name$q$, - local_schema, remote_col_table, remote_schema, remote_table); + local_schema, remote_col_table, + server_internal, remote_schema, remote_table, + remote_schema, remote_table); END $func$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- List the columns from a remote PG table +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, remote_table name, postgis_schema name DEFAULT 'public') +RETURNS TABLE(column_name name, column_type text) +AS $func$ +DECLARE + -- Import `geometry_columns` and geography_columns from the postgis schema + -- We assume that postgis is installed in the public schema + + -- Create local target schema if it does not exists + remote_geometry_view name := 'geometry_columns'; + remote_geography_view name := 'geography_columns'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema); +BEGIN + -- Import the foreign `geometry_columns` and `geography_columns` if not done + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_geometry_view + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I', + postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema); + END IF; + + BEGIN + -- Note: We return both the type and srid as the type + RETURN QUERY EXECUTE format($q$ + SELECT f_geometry_column::NAME as column_name, + type::TEXT || ',' || srid::TEXT as column_type + FROM + ( + SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I + ) _geo_views + WHERE + f_table_schema = '%s' AND + f_table_name = '%s' + $q$, + local_schema, remote_geometry_view, + local_schema, remote_geography_view, + remote_schema, remote_table); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Could not find Postgis installation in the remote "%" schema', postgis_schema; + RETURN; + END; +END +$func$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + -------------------------------------------------------------------------------- -- Public functions -------------------------------------------------------------------------------- @@ -170,7 +224,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( server TEXT, remote_schema TEXT, remote_table TEXT) -RETURNS TABLE(column_name name) +RETURNS TABLE(column_name name, column_type text) AS $$ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); @@ -178,7 +232,7 @@ DECLARE BEGIN CASE server_type WHEN 'postgres_fdw' THEN - RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema, remote_table); + RETURN QUERY SELECT * FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema, remote_table); ELSE RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; END CASE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index 07b0344..d9d263a 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -32,7 +32,7 @@ SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback2'::tex }'::jsonb); -- =================================================================== --- create objects used through FDW loopback server +-- Setup 1 -- =================================================================== \c cdb_fs_tester cdb_fs_tester @@ -92,7 +92,7 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopbac SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -- =================================================================== --- Cleanup +-- Cleanup 1 -- =================================================================== \set QUIET on @@ -105,6 +105,51 @@ DROP TABLE "S 1". "T 4"; DROP SCHEMA "S 1"; DROP TYPE user_enum; + +-- =================================================================== +-- Setup 2: Using Postgis too +-- =================================================================== + +\c cdb_fs_tester postgres + +CREATE EXTENSION postgis; + +\c cdb_fs_tester cdb_fs_tester + +CREATE SCHEMA "S 1"; +CREATE TABLE "S 1"."T 5" ( + geom geometry(Geometry,4326), + geom_wm geometry(GeometryZ,3857), + geo_nosrid geometry, + geog geography +); + +\c contrib_regression postgres +SET client_min_messages TO notice; +\set VERBOSITY terse +\set QUIET off + + +-- =================================================================== +-- Test the listing functions +-- =================================================================== + +\echo 'Test listing of remote geometry columns (sunny day)' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); +\echo 'Test listing of remote geometry columns (sunny day) - Rerun' +-- Rerun should be ok +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); + +-- =================================================================== +-- Cleanup 2 +-- =================================================================== +\set QUIET on + +\c cdb_fs_tester cdb_fs_tester +DROP TABLE "S 1". "T 5"; + +DROP SCHEMA "S 1"; + \c contrib_regression postgres \set QUIET on SET client_min_messages TO error; @@ -112,6 +157,7 @@ SET client_min_messages TO error; SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); + DROP DATABASE cdb_fs_tester; DROP ROLE cdb_fs_tester; DROP EXTENSION postgres_fdw; diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index f36517b..4554d3f 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -10,13 +10,24 @@ T 2 T 3 T 4 Test listing of remote columns (sunny day) -C 1 -c2 -c3 -c4 -c5 -c6 -c7 -c8 +NOTICE: Could not find Postgis installation in the remote "public" schema +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +Test listing of remote geometry columns (sunny day) +geo_nosrid|GEOMETRY,0 +geog|Geometry,0 +geom|GEOMETRY,4326 +geom_wm|GEOMETRY,3857 +Test listing of remote geometry columns (sunny day) - Rerun +geo_nosrid|GEOMETRY,0 +geog|Geometry,0 +geom|GEOMETRY,4326 +geom_wm|GEOMETRY,3857 D1| D2| From 57f420ab4c34751af20403999a177fbdbe333858 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Nov 2019 15:32:34 +0100 Subject: [PATCH 028/108] CDB_FederatedServerListRemote: Add rainy tests --- test/CDB_FederatedServerListRemote.sql | 16 ++++++++++++++++ test/CDB_FederatedServerListRemote_expect | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index d9d263a..189e032 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -85,12 +85,28 @@ SET client_min_messages TO notice; \echo 'Test listing of remote schemas (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\echo 'Test listing of remote schemas (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist'); + \echo 'Test listing of remote tables (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\echo 'Test listing of remote tables (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1'); +\echo 'Test listing of remote tables (rainy day): Remote schema does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist'); + \echo 'Test listing of remote columns (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\echo 'Test listing of remote columns (rainy day): Server does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1'); +\echo 'Test listing of remote columns (rainy day): Remote schema does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1'); +\echo 'Test listing of remote columns (rainy day): Remote table does not exist' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist'); + + -- =================================================================== -- Cleanup 1 -- =================================================================== diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index 4554d3f..6a2dc46 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -4,11 +4,16 @@ Test listing of remote schemas (sunny day) S 1 information_schema public +Test listing of remote schemas (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist Test listing of remote tables (sunny day) T 1 T 2 T 3 T 4 +Test listing of remote tables (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist +Test listing of remote tables (rainy day): Remote schema does not exist Test listing of remote columns (sunny day) NOTICE: Could not find Postgis installation in the remote "public" schema C 1|integer @@ -19,6 +24,10 @@ c5|timestamp without time zone c6|character varying c7|character c8|USER-DEFINED +Test listing of remote columns (rainy day): Server does not exist +ERROR: Server "Does Not Exist" does not exist +Test listing of remote columns (rainy day): Remote schema does not exist +Test listing of remote columns (rainy day): Remote table does not exist Test listing of remote geometry columns (sunny day) geo_nosrid|GEOMETRY,0 geog|Geometry,0 From 03bc3436cd58cef015b3c9b74e3f3b4d3b5cc1b1 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Nov 2019 16:29:18 +0100 Subject: [PATCH 029/108] Add more tests for [un-]registering servers --- scripts-available/CDB_FederatedServer.sql | 20 ++++--- test/CDB_FederatedServer.sql | 63 ++++++++++++++++++++--- test/CDB_FederatedServer_expect | 10 ++++ 3 files changed, 80 insertions(+), 13 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 324359f..43401f7 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -126,16 +126,21 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(inpu RETURNS jsonb AS $$ DECLARE - user_mapping jsonb := json_build_object( - 'user_mapping', - jsonb_build_object( 'user', input_config->'credentials'->'username', - 'password', input_config->'credentials'->'password') - ); + mapping jsonb := '{}'::jsonb; BEGIN IF NOT (input_config ? 'credentials') THEN RAISE EXCEPTION 'Credentials are mandatory'; END IF; - RETURN (input_config - 'credentials')::jsonb || user_mapping; + + -- For now, allow not passing username or password + IF input_config->'credentials'->'username' IS NOT NULL THEN + mapping := jsonb_build_object('user', input_config->'credentials'->'username'); + END IF; + IF input_config->'credentials'->'password' IS NOT NULL THEN + mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password'); + END IF; + + RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping); END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; @@ -154,6 +159,9 @@ DECLARE }'; server_config jsonb; BEGIN + IF NOT (input_config ? 'server') THEN + RAISE EXCEPTION 'Server information is mandatory'; + END IF; server_config := default_options || to_jsonb(input_config->'server'); RETURN jsonb_set(input_config, '{server}'::text[], server_config); END diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 58a033b..ffce997 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -71,15 +71,64 @@ SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist':: SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); --- Should show the appropiate output (database, read-write, user, pass) +-- Test empty config +SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{}'); +-- Test without passing credentials +SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + } +}'::jsonb); +-- Test with empty credentials +SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ + "server": { + "dbname": "fdw_target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { } +}'::jsonb); +SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); +SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server := 'empty'::text); +-- Test without without server options +SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ + "credentials": { + "username": "other_remote_user", + "password": "foobarino" + } +}'::jsonb); --- Test multiple user mappings +-- Should work ok with special characters +SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or''not'::text, config := '{ + "server": { + "dbname": "fdw target", + "host": "localhost", + "port": @@PGPORT@@, + "extensions": "postgis", + "updatable": "false", + "use_remote_estimate": "true", + "fetch_size": "1000" + }, + "credentials": { + "username": "fdw user", + "password": "foo barino" + } +}'::jsonb); +SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers(); +SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote" or''not'::text); --- Should work ok with special characters in the name - --- Should throw with invalid or NULL config - --- -- List works with multiple and single server +-- Should throw when trying to unregistering a server that doesn't exists +SELECT '8.4', cartodb.CDB_Federated_Server_Unregister(server := 'Does not exist'::text); -- Cleanup \set QUIET on diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index 24c6479..68972b1 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -11,3 +11,13 @@ 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) ERROR: Server "doesNotExist" does not exist 6.1| +ERROR: Server information is mandatory +ERROR: Credentials are mandatory +7.3| +7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,) +7.5| +ERROR: Server information is mandatory +8.1| +8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") +8.3| +ERROR: Server "Does not exist" does not exist From 0f66c54ce5564355392667b9c31948312c5ebd83 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Nov 2019 16:53:48 +0100 Subject: [PATCH 030/108] Comment cleanup --- scripts-available/CDB_FederatedServer.sql | 2 -- scripts-available/CDB_FederatedServerListRemote.sql | 5 ++++- scripts-available/CDB_Groups_API.sql | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 43401f7..f5971d5 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -278,8 +278,6 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List registered servers -- --- TODO: Decide whether we want to show extra config (extensions, fetch_size, use_remote_estimate) --- TODO: Handle multiple user mappings in the same server CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%') RETURNS TABLE ( name text, diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 7cc3fb4..0b00f24 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -133,7 +133,7 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG RETURNS TABLE(column_name name, column_type text) AS $func$ DECLARE - -- Import `geometry_columns` and geography_columns from the postgis schema + -- Import `geometry_columns` and `geography_columns` from the postgis schema -- We assume that postgis is installed in the public schema -- Create local target schema if it does not exists @@ -220,6 +220,9 @@ END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- List the columns of a remote table in a federated server that the current user has access to. +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( server TEXT, remote_schema TEXT, diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index 660d4e8..b29eb77 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -5,7 +5,6 @@ -- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }'); ---------------------------------- --- TODO: delete this development cleanup before final merge DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text); DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text); From 10ed64789d7e13d282ea3098b9c5f32468b8569e Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Nov 2019 19:23:36 +0100 Subject: [PATCH 031/108] Add functions to grant and revoke permissions over federated servers --- scripts-available/CDB_FederatedServer.sql | 70 ++++++++++++++++++++--- test/CDB_FederatedServer.sql | 60 +++++++++++++++++++ test/CDB_FederatedServer_expect | 15 +++++ 3 files changed, 137 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index f5971d5..b3b821a 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -26,15 +26,20 @@ AS $$ DECLARE internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); BEGIN - -- We discard anything that would be truncated - IF (char_length(internal_server_name) < 64) THEN - IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN - RAISE EXCEPTION 'Server "%" does not exist', input_name; - END IF; - RETURN internal_server_name::name; - ELSE - RAISE EXCEPTION 'Server name is too long to be used as identifier'; + IF input_name IS NULL THEN + RAISE EXCEPTION 'Server name cannot be NULL'; END IF; + + -- We discard anything that would be truncated + IF (char_length(internal_server_name) >= 64) THEN + RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name; + END IF; + + IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN + RAISE EXCEPTION 'Server "%" does not exist', input_name; + END IF; + + RETURN internal_server_name::name; END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; @@ -315,3 +320,52 @@ BEGIN END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; + + +-- +-- Grant access to a server +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, usernames text[]) +RETURNS void +AS $$ +DECLARE + server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + user_role TEXT; + username TEXT; +BEGIN + FOREACH username IN ARRAY usernames + LOOP + user_role := @extschema@._CDB_User_RoleFromUsername(username); + IF (user_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" does not exists', username; + END IF; + EXECUTE format('GRANT %I TO %I', server_role_name, user_role); + END loop; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Revoke access to a server +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, usernames text[]) +RETURNS void +AS $$ +DECLARE + server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + user_role TEXT; + username TEXT; +BEGIN + FOREACH username IN ARRAY usernames + LOOP + user_role := @extschema@._CDB_User_RoleFromUsername(username); + IF (user_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" does not exists', username; + END IF; + EXECUTE format('REVOKE %I FROM %I', server_role_name, user_role); + END loop; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index ffce997..69c07a5 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -72,6 +72,7 @@ SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::tex SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); -- Test empty config +SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server := NULL::text, config := '{ "server": {}, "credentials" : {}}'); SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{}'); -- Test without passing credentials SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ @@ -130,8 +131,67 @@ SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote" or''n -- Should throw when trying to unregistering a server that doesn't exists SELECT '8.4', cartodb.CDB_Federated_Server_Unregister(server := 'Does not exist'::text); +-- Test permissions +\set QUIET on + +-- We create a username following the same steps as organization members +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; +CREATE SCHEMA cdb_fs_tester AUTHORIZATION cdb_fs_tester; +SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester'); +ALTER ROLE cdb_fs_tester SET search_path TO cdb_fs_tester,cartodb,public; + +\set QUIET off + +SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); + +\c contrib_regression cdb_fs_tester + +-- A normal user can list existing servers +SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); +-- Creating a server without superadmin should fail +SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); + + +\c contrib_regression postgres + +SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); +SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', usernames := ARRAY['cdb_fs_tester']); +SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['does not exist']); + +-- Grant again raises a notice +SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); + +-- Revoke works +SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); +SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); + +-- Dropping the server without revoking access works +SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote3'::text); + -- Cleanup \set QUIET on +DROP SCHEMA cdb_fs_tester CASCADE; +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; DROP EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index 68972b1..3c11692 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -11,6 +11,7 @@ 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) ERROR: Server "doesNotExist" does not exist 6.1| +ERROR: Server name cannot be NULL ERROR: Server information is mandatory ERROR: Credentials are mandatory 7.3| @@ -21,3 +22,17 @@ ERROR: Server information is mandatory 8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") 8.3| ERROR: Server "Does not exist" does not exist + +9.1| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,) +ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw +You are now connected to database "contrib_regression" as user "postgres". +9.5| +ERROR: Server "does not exist" does not exist +ERROR: User role "does not exist" does not exists +NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" +9.8| +9.9| +9.10| +9.11| From 98b2b02199846750ea918823bc3ebde9fd9e2c1e Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 11:19:50 +0100 Subject: [PATCH 032/108] Improve tests readability and test deleting a server from granted access --- scripts-available/CDB_FederatedServer.sql | 10 +++- test/CDB_FederatedServer.sql | 70 ++++++++++++++++------- test/CDB_FederatedServer_expect | 32 ++++++++++- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index b3b821a..67ca036 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -273,9 +273,13 @@ DECLARE server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN - EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); - EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); - EXECUTE FORMAT ('DROP ROLE %I', role_name); + BEGIN + EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); + EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); + EXECUTE FORMAT ('DROP ROLE %I', role_name); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Unnecessary permissions to drop the server "%": %', server, SQLERRM; + END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 69c07a5..5371235 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -6,9 +6,13 @@ SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; \set QUIET off --- Register a new server +\echo "List empty servers shows nothing" SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); + +\echo "List non-existent server shows nothing" SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server := 'doesNotExist'); + +\echo "Create and list a server works" SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::text, config := '{ "server": { "host": "localhost", @@ -21,7 +25,7 @@ SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::tex }'::jsonb); SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); --- Register a second server +\echo "Create and list a second server works" SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ "server": { "dbname": "fdw_target", @@ -38,11 +42,12 @@ SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te } }'::jsonb); SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); --- Check that CDB_Federated_Server_List_Servers works with name + +\echo "List server by name works" SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server := 'myRemote'); --- Re-register the second server +\echo "Re-register a second server works" SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ "server": { "dbname": "fdw_target", @@ -60,21 +65,24 @@ SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te }'::jsonb); SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); --- Unregister #1 +\echo "Unregister server 1 works" SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote'::text); SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); --- Unregister a server that doesn't exist +\echo "Unregistering a server that doesn't exist fails" SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist'::text); --- Unregister #2 +\echo "Unregister the second server works" SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); --- Test empty config +\echo "Create a server with NULL name fails" SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server := NULL::text, config := '{ "server": {}, "credentials" : {}}'); +\echo "Create a server with NULL config fails" +SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := NULL::jsonb); +\echo "Create a server with empty config fails" SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{}'); --- Test without passing credentials +\echo "Create a server without credentials fails" SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "server": { "dbname": "fdw_target", @@ -86,7 +94,7 @@ SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, "fetch_size": "1000" } }'::jsonb); --- Test with empty credentials +\echo "Create a server with empty credentials works" SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "server": { "dbname": "fdw_target", @@ -101,7 +109,7 @@ SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, }'::jsonb); SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server := 'empty'::text); --- Test without without server options +\echo "Create a server without options fails" SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "credentials": { "username": "other_remote_user", @@ -109,7 +117,7 @@ SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, } }'::jsonb); --- Should work ok with special characters +\echo "Create a server with special characters works" SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or''not'::text, config := '{ "server": { "dbname": "fdw target", @@ -128,9 +136,6 @@ SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or'' SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers(); SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote" or''not'::text); --- Should throw when trying to unregistering a server that doesn't exists -SELECT '8.4', cartodb.CDB_Federated_Server_Unregister(server := 'Does not exist'::text); - -- Test permissions \set QUIET on @@ -156,9 +161,10 @@ SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::te \c contrib_regression cdb_fs_tester --- A normal user can list existing servers +\echo "All users are able to list servers" SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); --- Creating a server without superadmin should fail + +\echo "Only superadmins can create servers" SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ "server": { "host": "localhost", @@ -173,20 +179,42 @@ SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::te \c contrib_regression postgres +\echo "Granting access to a user works" SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', usernames := ARRAY['cdb_fs_tester']); SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['does not exist']); --- Grant again raises a notice +\echo "Granting access again raises a notice" SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); --- Revoke works +\echo "Revoking access to a user works" SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); --- Dropping the server without revoking access works +\echo "Unregistering a server with active grants works" SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote3'::text); + +\echo "A user with granted access can't drop a server" +SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "fdw_user", + "password": "foobarino" + } +}'::jsonb); +SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote4', usernames := ARRAY['cdb_fs_tester']); + +\c contrib_regression cdb_fs_tester +SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::text); + +\c contrib_regression postgres +SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::text); + + -- Cleanup \set QUIET on DROP SCHEMA cdb_fs_tester CASCADE; @@ -194,5 +222,3 @@ REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; DROP ROLE cdb_fs_tester; DROP EXTENSION postgres_fdw; \set QUIET off - - diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index 3c11692..290c234 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -1,38 +1,68 @@ +"List empty servers shows nothing" +"List non-existent server shows nothing" +"Create and list a server works" 1.3| 1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +"Create and list a second server works" 2.1| 2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) +"List server by name works" 2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) +"Re-register a second server works" 3.1| 3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) +"Unregister server 1 works" 4.1| 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) +"Unregistering a server that doesn't exist fails" ERROR: Server "doesNotExist" does not exist +"Unregister the second server works" 6.1| +"Create a server with NULL name fails" ERROR: Server name cannot be NULL +"Create a server with NULL config fails" +7.01| +"Create a server with empty config fails" ERROR: Server information is mandatory +"Create a server without credentials fails" ERROR: Credentials are mandatory +"Create a server with empty credentials works" 7.3| 7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,) 7.5| +"Create a server without options fails" ERROR: Server information is mandatory +"Create a server with special characters works" 8.1| 8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") 8.3| -ERROR: Server "Does not exist" does not exist 9.1| You are now connected to database "contrib_regression" as user "cdb_fs_tester". +"All users are able to list servers" 9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,) +"Only superadmins can create servers" ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw You are now connected to database "contrib_regression" as user "postgres". +"Granting access to a user works" 9.5| ERROR: Server "does not exist" does not exist ERROR: User role "does not exist" does not exists +"Granting access again raises a notice" NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" 9.8| +"Revoking access to a user works" 9.9| 9.10| +"Unregistering a server with active grants works" 9.11| +"A user with granted access can't drop a server" +10.1| +10.2| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +WARNING: no privileges could be revoked for "postgres_fdw" +ERROR: Unnecessary permissions to drop the server "myRemote4": permission denied to drop role +You are now connected to database "contrib_regression" as user "postgres". +10.4| From bac7d79ab341c610310e1b4114e6a56cea6aff25 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 13:22:45 +0100 Subject: [PATCH 033/108] CDB_FederatedServer: Improve tests readability --- scripts-available/CDB_FederatedServer.sql | 3 +- test/CDB_FederatedServer.sql | 46 ++++++++++----------- test/CDB_FederatedServer_expect | 49 +++++++++++------------ 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 67ca036..3c822b7 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -273,12 +273,13 @@ DECLARE server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN + SET client_min_messages = ERROR; BEGIN EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Unnecessary permissions to drop the server "%": %', server, SQLERRM; + RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; END; END $$ diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 5371235..a818f8e 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -6,13 +6,13 @@ SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; \set QUIET off -\echo "List empty servers shows nothing" +\echo '## List empty servers shows nothing' SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); -\echo "List non-existent server shows nothing" +\echo '## List non-existent server shows nothing' SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server := 'doesNotExist'); -\echo "Create and list a server works" +\echo '## Create and list a server works' SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::text, config := '{ "server": { "host": "localhost", @@ -25,7 +25,7 @@ SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::tex }'::jsonb); SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); -\echo "Create and list a second server works" +\echo '## Create and list a second server works' SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ "server": { "dbname": "fdw_target", @@ -43,11 +43,11 @@ SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te }'::jsonb); SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); -\echo "List server by name works" +\echo '## List server by name works' SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server := 'myRemote'); -\echo "Re-register a second server works" +\echo '## Re-register a second server works' SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ "server": { "dbname": "fdw_target", @@ -65,24 +65,24 @@ SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te }'::jsonb); SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); -\echo "Unregister server 1 works" +\echo '## Unregister server 1 works' SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote'::text); SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); -\echo "Unregistering a server that doesn't exist fails" +\echo '## Unregistering a server that does not exist fails' SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist'::text); -\echo "Unregister the second server works" +\echo '## Unregister the second server works' SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); -\echo "Create a server with NULL name fails" +\echo '## Create a server with NULL name fails' SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server := NULL::text, config := '{ "server": {}, "credentials" : {}}'); -\echo "Create a server with NULL config fails" +\echo '## Create a server with NULL config fails' SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := NULL::jsonb); -\echo "Create a server with empty config fails" +\echo '## Create a server with empty config fails' SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{}'); -\echo "Create a server without credentials fails" +\echo '## Create a server without credentials fails' SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "server": { "dbname": "fdw_target", @@ -94,7 +94,7 @@ SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, "fetch_size": "1000" } }'::jsonb); -\echo "Create a server with empty credentials works" +\echo '## Create a server with empty credentials works' SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "server": { "dbname": "fdw_target", @@ -109,7 +109,7 @@ SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, }'::jsonb); SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server := 'empty'::text); -\echo "Create a server without options fails" +\echo '## Create a server without options fails' SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ "credentials": { "username": "other_remote_user", @@ -117,7 +117,7 @@ SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, } }'::jsonb); -\echo "Create a server with special characters works" +\echo '## Create a server with special characters works' SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or''not'::text, config := '{ "server": { "dbname": "fdw target", @@ -161,10 +161,10 @@ SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::te \c contrib_regression cdb_fs_tester -\echo "All users are able to list servers" +\echo '## All users are able to list servers' SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); -\echo "Only superadmins can create servers" +\echo '## Only superadmins can create servers' SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ "server": { "host": "localhost", @@ -179,23 +179,23 @@ SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::te \c contrib_regression postgres -\echo "Granting access to a user works" +\echo '## Granting access to a user works' SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', usernames := ARRAY['cdb_fs_tester']); SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['does not exist']); -\echo "Granting access again raises a notice" +\echo '## Granting access again raises a notice' SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); -\echo "Revoking access to a user works" +\echo '## Revoking access to a user works' SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); -\echo "Unregistering a server with active grants works" +\echo '## Unregistering a server with active grants works' SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote3'::text); -\echo "A user with granted access can't drop a server" +\echo '## A user with granted access can not drop a server' SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ "server": { "host": "localhost", diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index 290c234..aecffa1 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -1,68 +1,67 @@ -"List empty servers shows nothing" -"List non-existent server shows nothing" -"Create and list a server works" +## List empty servers shows nothing +## List non-existent server shows nothing +## Create and list a server works 1.3| 1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -"Create and list a second server works" +## Create and list a second server works 2.1| 2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user) -"List server by name works" +## List server by name works 2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) -"Re-register a second server works" +## Re-register a second server works 3.1| 3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user) 3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) -"Unregister server 1 works" +## Unregister server 1 works 4.1| 4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user) -"Unregistering a server that doesn't exist fails" +## Unregistering a server that does not exist fails ERROR: Server "doesNotExist" does not exist -"Unregister the second server works" +## Unregister the second server works 6.1| -"Create a server with NULL name fails" +## Create a server with NULL name fails ERROR: Server name cannot be NULL -"Create a server with NULL config fails" +## Create a server with NULL config fails 7.01| -"Create a server with empty config fails" +## Create a server with empty config fails ERROR: Server information is mandatory -"Create a server without credentials fails" +## Create a server without credentials fails ERROR: Credentials are mandatory -"Create a server with empty credentials works" +## Create a server with empty credentials works 7.3| 7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,) 7.5| -"Create a server without options fails" +## Create a server without options fails ERROR: Server information is mandatory -"Create a server with special characters works" +## Create a server with special characters works 8.1| 8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") 8.3| 9.1| You are now connected to database "contrib_regression" as user "cdb_fs_tester". -"All users are able to list servers" +## All users are able to list servers 9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,) -"Only superadmins can create servers" +## Only superadmins can create servers ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw You are now connected to database "contrib_regression" as user "postgres". -"Granting access to a user works" +## Granting access to a user works 9.5| ERROR: Server "does not exist" does not exist ERROR: User role "does not exist" does not exists -"Granting access again raises a notice" +## Granting access again raises a notice NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" 9.8| -"Revoking access to a user works" +## Revoking access to a user works 9.9| 9.10| -"Unregistering a server with active grants works" +## Unregistering a server with active grants works 9.11| -"A user with granted access can't drop a server" +## A user with granted access can not drop a server 10.1| 10.2| You are now connected to database "contrib_regression" as user "cdb_fs_tester". -WARNING: no privileges could be revoked for "postgres_fdw" -ERROR: Unnecessary permissions to drop the server "myRemote4": permission denied to drop role +ERROR: Not enough permissions to drop the server "myRemote4" You are now connected to database "contrib_regression" as user "postgres". 10.4| From aa481f4219e8f36d6525668ec3544f98a4147db5 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 14:35:52 +0100 Subject: [PATCH 034/108] Permission tests and fixes for CDB_FederatedServerListRemote --- scripts-available/CDB_FederatedServer.sql | 37 +++-- .../CDB_FederatedServerListRemote.sql | 15 +- test/CDB_FederatedServerListRemote.sql | 153 ++++++++++++++++-- test/CDB_FederatedServerListRemote_expect | 134 +++++++++++++-- 4 files changed, 297 insertions(+), 42 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 3c822b7..1d55198 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -98,12 +98,17 @@ DECLARE schema_name text := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); BEGIN + -- By changing the local role to the owner of the server we have an + -- easy way to check for permissions and keep all objects under the same owner + BEGIN + EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); + END; + IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN - BEGIN - EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); - EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'TODO: This needs a better error handling after reviewing permissions: %', SQLERRM; - END; + EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name); END IF; RETURN schema_name; END @@ -219,13 +224,19 @@ BEGIN IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN BEGIN EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); - EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); + -- TODO: Delete this IF before merging to make sure nobody creates a role + -- that is later used automatically by us granting them all permissions in the foreign server + -- TODO: This is here to help debugging during development (so failures to destroy objects are allowed) + -- TODO + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN + EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); + END IF; + EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); - -- NOTE: we use a PUBLIC user mapping but control access to the SERVER - -- so that we don't need to create a mapping for every user nor store credentials elsewhere - EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', server_internal); + EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM USING HINT = 'Please clean the left over objects'; @@ -251,12 +262,12 @@ BEGIN LOOP IF NOT EXISTS ( WITH a AS ( - SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = 'public' + SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name ) SELECT * from a where options = option.key) THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value); ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value); END IF; END LOOP; END @@ -275,7 +286,7 @@ DECLARE BEGIN SET client_min_messages = ERROR; BEGIN - EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', server_internal); + EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 0b00f24..09fb678 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -34,11 +34,16 @@ BEGIN END IF; -- Return the result we're interested in. Exclude toast and temp schemas - RETURN QUERY EXECUTE format(' - SELECT schema_name::name AS remote_schema FROM %I.%I - WHERE schema_name NOT LIKE %s - ORDER BY remote_schema - ', local_schema, remote_table, '''pg_%'''); + BEGIN + RETURN QUERY EXECUTE format(' + SELECT schema_name::name AS remote_schema FROM %I.%I + WHERE schema_name NOT LIKE %s + ORDER BY remote_schema + ', local_schema, remote_table, '''pg_%'''); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%": %', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal), SQLERRM; + END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index 189e032..f72fb42 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -6,9 +6,24 @@ SET client_min_messages TO error; \set VERBOSITY terse SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; -CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; + +-- We create a username following the same steps as organization members +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; +CREATE SCHEMA cdb_fs_tester AUTHORIZATION cdb_fs_tester; +SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester'); +ALTER ROLE cdb_fs_tester SET search_path TO cdb_fs_tester,cartodb,public; + +CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; +CREATE SCHEMA cdb_fs_tester2 AUTHORIZATION cdb_fs_tester2; +SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester2'); +ALTER ROLE cdb_fs_tester2 SET search_path TO cdb_fs_tester2,cartodb,public; + +-- Create database to be used as remote CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ "server": { "host": "localhost", @@ -80,33 +95,139 @@ SET client_min_messages TO notice; -- =================================================================== --- Test the listing functions +-- Test listing remote schemas -- =================================================================== -\echo 'Test listing of remote schemas (sunny day)' +\echo '## Test listing of remote schemas without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); -\echo 'Test listing of remote schemas (rainy day): Server does not exist' +\echo '## Test listing of remote schemas without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); +\c contrib_regression postgres + +\echo '## Test listing of remote schemas (rainy day): Server does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist'); -\echo 'Test listing of remote tables (sunny day)' + +-- =================================================================== +-- Test listing remote tables +-- =================================================================== + +\echo '## Test listing of remote tables without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -\echo 'Test listing of remote tables (rainy day): Server does not exist' +\echo '## Test listing of remote tables without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote tables (rainy day): Server does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1'); -\echo 'Test listing of remote tables (rainy day): Remote schema does not exist' + +\echo '## Test listing of remote tables (rainy day): Remote schema does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist'); -\echo 'Test listing of remote columns (sunny day)' + +-- =================================================================== +-- Test listing remote columns +-- =================================================================== + +\echo '## Test listing of remote columns without permissions before the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); -\echo 'Test listing of remote columns (rainy day): Server does not exist' +\echo '## Test listing of remote columns without permissions after the first instantiation (rainy day)' +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns without permissions after revoking access (rainy day)' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); +\c contrib_regression postgres + +\echo '## Test listing of remote columns (rainy day): Server does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1'); -\echo 'Test listing of remote columns (rainy day): Remote schema does not exist' + +\echo '## Test listing of remote columns (rainy day): Remote schema does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1'); -\echo 'Test listing of remote columns (rainy day): Remote table does not exist' + +\echo '## Test listing of remote columns (rainy day): Remote table does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist'); +-- =================================================================== +-- Test that using a different user to list tables and dropping it +-- does not break the server: We use loopback2 as it's in a clean state +-- =================================================================== + + +\echo '## Test listing of remote objects with permissions (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', usernames := ARRAY['cdb_fs_tester2']); +\c contrib_regression cdb_fs_tester2 +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); + +\c contrib_regression postgres +\echo '## Test that dropping the granted user works fine (sunny day)' +DROP SCHEMA cdb_fs_tester2 CASCADE; +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; +DROP ROLE cdb_fs_tester2; + +\echo '## Test listing of remote objects with other user still works (sunny day)' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', usernames := ARRAY['cdb_fs_tester']); +\c contrib_regression cdb_fs_tester +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1'); + + -- =================================================================== -- Cleanup 1 -- =================================================================== @@ -150,9 +271,9 @@ SET client_min_messages TO notice; -- Test the listing functions -- =================================================================== -\echo 'Test listing of remote geometry columns (sunny day)' +\echo '## Test listing of remote geometry columns (sunny day)' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); -\echo 'Test listing of remote geometry columns (sunny day) - Rerun' +\echo '## Test listing of remote geometry columns (sunny day) - Rerun' -- Rerun should be ok SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); @@ -175,6 +296,12 @@ SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text) SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); DROP DATABASE cdb_fs_tester; + +-- Drop role +DROP SCHEMA cdb_fs_tester CASCADE; +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; DROP ROLE cdb_fs_tester; + DROP EXTENSION postgres_fdw; + \set QUIET off diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index 6a2dc46..3228992 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -1,20 +1,67 @@ + + C1| C2| -Test listing of remote schemas (sunny day) +## Test listing of remote schemas without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas (sunny day) S 1 information_schema public -Test listing of remote schemas (rainy day): Server does not exist +## Test listing of remote schemas without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +S 1 +information_schema +public +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote schemas (rainy day): Server does not exist ERROR: Server "Does Not Exist" does not exist -Test listing of remote tables (sunny day) +## Test listing of remote tables without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables (sunny day) T 1 T 2 T 3 T 4 -Test listing of remote tables (rainy day): Server does not exist +## Test listing of remote tables without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +T 1 +T 2 +T 3 +T 4 +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote tables (rainy day): Server does not exist ERROR: Server "Does Not Exist" does not exist -Test listing of remote tables (rainy day): Remote schema does not exist -Test listing of remote columns (sunny day) +## Test listing of remote tables (rainy day): Remote schema does not exist +## Test listing of remote columns without permissions before the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns (sunny day) NOTICE: Could not find Postgis installation in the remote "public" schema C 1|integer c2|integer @@ -24,16 +71,81 @@ c5|timestamp without time zone c6|character varying c7|character c8|USER-DEFINED -Test listing of remote columns (rainy day): Server does not exist +## Test listing of remote columns without permissions after the first instantiation (rainy day) +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +NOTICE: Could not find Postgis installation in the remote "public" schema +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns without permissions after revoking access (rainy day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +You are now connected to database "contrib_regression" as user "postgres". +## Test listing of remote columns (rainy day): Server does not exist ERROR: Server "Does Not Exist" does not exist -Test listing of remote columns (rainy day): Remote schema does not exist -Test listing of remote columns (rainy day): Remote table does not exist -Test listing of remote geometry columns (sunny day) +## Test listing of remote columns (rainy day): Remote schema does not exist +## Test listing of remote columns (rainy day): Remote table does not exist +## Test listing of remote objects with permissions (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +S 1 +information_schema +public +T 1 +T 2 +T 3 +T 4 +NOTICE: Could not find Postgis installation in the remote "public" schema +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +You are now connected to database "contrib_regression" as user "postgres". +## Test that dropping the granted user works fine (sunny day) +DROP SCHEMA +REVOKE +DROP ROLE +## Test listing of remote objects with other user still works (sunny day) + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +S 1 +information_schema +public +T 1 +T 2 +T 3 +T 4 +NOTICE: Could not find Postgis installation in the remote "public" schema +C 1|integer +c2|integer +c3|text +c4|timestamp with time zone +c5|timestamp without time zone +c6|character varying +c7|character +c8|USER-DEFINED +## Test listing of remote geometry columns (sunny day) geo_nosrid|GEOMETRY,0 geog|Geometry,0 geom|GEOMETRY,4326 geom_wm|GEOMETRY,3857 -Test listing of remote geometry columns (sunny day) - Rerun +## Test listing of remote geometry columns (sunny day) - Rerun geo_nosrid|GEOMETRY,0 geog|Geometry,0 geom|GEOMETRY,4326 From 73ee4107d4e30be4b675f4a4aafdce5538313c10 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 14:48:52 +0100 Subject: [PATCH 035/108] CDB_FederatedServerTables: Improve tests readability --- test/CDB_FederatedServerTables.sql | 18 +++++++++++++++++- test/CDB_FederatedServerTables_expect | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 460252d..7ed7630 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -54,6 +54,7 @@ SET client_min_messages TO error; \set VERBOSITY terse \set QUIET off +\echo '## Registering an existing table works' SELECT 'R1', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -70,6 +71,7 @@ Select 'list_remotes1', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); +\echo '## Registering another existing table works' SELECT 'R2', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -88,7 +90,8 @@ Select 'list_remotes2', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); --- Reregistering R2 with different parameters should work + +\echo '## Re-registering a table works' SELECT 'R3', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -103,6 +106,7 @@ SELECT 'S3_old', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTabl -- And the new appear SELECT 'S3_new', cartodb_id, ST_AsText(the_geom), another_field FROM different_name; +\echo '## Unregistering works' -- Deregistering the first table SELECT 'U1', CDB_Federated_Table_Unregister( server => 'loopback', @@ -117,6 +121,18 @@ Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); +-- Try to register with invalid / NULL server +-- Try to register with invalid / NULL schema +-- Try to register with invalid / NULL table +-- Try to register with invalid / NULL id +-- Try to register with invalid / NULL geom_column +-- Try to register with invalid / NULL webmercator_column +-- Try to register the same table twice (different name) should fail +-- Check that conflict is handled nicely (target view already exists) + +-- Try permissions tricks + + -- =================================================================== -- Cleanup -- =================================================================== diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 21229d3..7298a72 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -1,4 +1,5 @@ C1| +## Registering an existing table works R1| V1| SELECT t.id AS cartodb_id, t.geom AS the_geom, @@ -8,6 +9,7 @@ V1| SELECT t.id AS cartodb_id, S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 list_remotes1|(remote_geom,public.remote_geom) +## Registering another existing table works R2| V2| SELECT t.id AS cartodb_id, t.geom AS the_geom, @@ -17,9 +19,11 @@ V2| SELECT t.id AS cartodb_id, S2|3|POINT(3 3)|patata list_remotes2|(remote_geom2,"public.""myFullTable""") list_remotes2|(remote_geom,public.remote_geom) +## Re-registering a table works R3| ERROR: relation "myFullTable" does not exist at character 70 S3_new|3|POINT(3 3)|patata +## Unregistering works U1| ERROR: relation "remote_geom" does not exist at character 71 list_remotes3|(remote_geom2,public.different_name) From e2beefdb53d9634b1f0561d4839a3a1a5d08611b Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 16:32:07 +0100 Subject: [PATCH 036/108] Test around NULLs and invalids arguments --- scripts-available/CDB_FederatedServer.sql | 12 +- .../CDB_FederatedServerListRemote.sql | 8 +- .../CDB_FederatedServerTables.sql | 4 + test/CDB_FederatedServerListRemote.sql | 3 + test/CDB_FederatedServerListRemote_expect | 2 + test/CDB_FederatedServerTables.sql | 130 +++++++++++++++++- test/CDB_FederatedServerTables_expect | 26 ++++ 7 files changed, 177 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 1d55198..0e113a7 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -64,9 +64,11 @@ RETURNS NAME AS $$ DECLARE hash_value text := md5(internal_server_name::text || '__' || schema_name::text); - schema_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value); BEGIN - RETURN schema_name::name; + IF schema_name IS NULL THEN + RAISE EXCEPTION 'Schema name cannot be NULL'; + END IF; + RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name; END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; @@ -102,7 +104,11 @@ BEGIN -- easy way to check for permissions and keep all objects under the same owner BEGIN EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); - EXCEPTION WHEN OTHERS THEN + EXCEPTION + WHEN invalid_parameter_value THEN + RAISE EXCEPTION 'Server "%" does not exist', + @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); + WHEN OTHERS THEN RAISE EXCEPTION 'Not enough permissions to access the server "%"', @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name); END; diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 09fb678..d8932c0 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -41,8 +41,8 @@ BEGIN ORDER BY remote_schema ', local_schema, remote_table, '''pg_%'''); EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Not enough permissions to access the server "%": %', - @extschema@.__CDB_FS_Extract_Server_Name(server_internal), SQLERRM; + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); END; END $$ @@ -238,6 +238,10 @@ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN + IF remote_table IS NULL THEN + RAISE EXCEPTION 'Remote table name cannot be NULL'; + END IF; + CASE server_type WHEN 'postgres_fdw' THEN RETURN QUERY SELECT * FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema, remote_table); diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index bd93445..db43d1c 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -104,6 +104,10 @@ DECLARE webmercator_expression TEXT; carto_columns_expression TEXT[]; BEGIN + IF remote_table IS NULL THEN + RAISE EXCEPTION 'Remote table name cannot be NULL'; + END IF; + -- Use geom_column as default for webmercator_column IF webmercator_column IS NULL THEN webmercator_column := geom_column; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index f72fb42..e230ab0 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -200,6 +200,9 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \echo '## Test listing of remote columns (rainy day): Remote table does not exist' SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist'); +\echo '## Test listing of remote columns (rainy day): Remote table is NULL' +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => NULL::text); + -- =================================================================== -- Test that using a different user to list tables and dropping it diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index 3228992..e05af3a 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -97,6 +97,8 @@ You are now connected to database "contrib_regression" as user "postgres". ERROR: Server "Does Not Exist" does not exist ## Test listing of remote columns (rainy day): Remote schema does not exist ## Test listing of remote columns (rainy day): Remote table does not exist +## Test listing of remote columns (rainy day): Remote table is NULL +ERROR: Remote table name cannot be NULL ## Test listing of remote objects with permissions (sunny day) You are now connected to database "contrib_regression" as user "cdb_fs_tester2". diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 7ed7630..604dcfd 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -4,6 +4,7 @@ \set QUIET on SET client_min_messages TO error; \set VERBOSITY terse + SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; @@ -121,9 +122,126 @@ Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); --- Try to register with invalid / NULL server --- Try to register with invalid / NULL schema --- Try to register with invalid / NULL table +\echo '## Registering a table: Invalid server fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'Does not exist', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL server fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => NULL::text, + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid schema fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'Does not exist', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL schema fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => NULL::text, + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid table fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'Does not exist', + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL table fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => NULL::text, + id_column => 'id', + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid id fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'Does not exist', + geom_column => 'geom' + ); + +\echo '## Registering a table: NULL id fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => NULL::text, + geom_column => 'geom' + ); + +\echo '## Registering a table: Invalid geom_column fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'Does not exists' + ); + +\echo '## Registering a table: NULL geom_column is OK' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => NULL::text + ); +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + +\echo '## Registering a table: Invalid webmercator_column fails' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + webmercator_column => 'Does not exists' + ); + +\echo '## Registering a table: NULL webmercator_column is OK' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + webmercator_column => NULL::text + ); +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + -- Try to register with invalid / NULL id -- Try to register with invalid / NULL geom_column -- Try to register with invalid / NULL webmercator_column @@ -132,6 +250,12 @@ Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( -- Try permissions tricks +-- Try registering and accessing a table as normal user + +-- Try register with one user and reading it with other +-- Try register with one user and deleting it with another + + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 7298a72..497c057 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -27,4 +27,30 @@ S3_new|3|POINT(3 3)|patata U1| ERROR: relation "remote_geom" does not exist at character 71 list_remotes3|(remote_geom2,public.different_name) +## Registering a table: Invalid server fails +ERROR: Server "Does not exist" does not exist +## Registering a table: NULL server fails +ERROR: Server name cannot be NULL +## Registering a table: Invalid schema fails +ERROR: schema "Does not exist" is not present on foreign server "cdb_fs_loopback" +## Registering a table: NULL schema fails +ERROR: Schema name cannot be NULL +## Registering a table: Invalid table fails +ERROR: relation "cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.Does not exist" does not exist +## Registering a table: NULL table fails +ERROR: Remote table name cannot be NULL +## Registering a table: Invalid id fails +ERROR: non integer id_column "Does not exist" +## Registering a table: NULL id fails +ERROR: non integer id_column "" +## Registering a table: Invalid geom_column fails +ERROR: non geometry column "Does not exists" +## Registering a table: NULL geom_column is OK + + +## Registering a table: Invalid webmercator_column fails +ERROR: non geometry column "Does not exists" +## Registering a table: NULL webmercator_column is OK + + D1| From c9b01592e795e38ec4b4dbe74a456597d1a02178 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 16:53:25 +0100 Subject: [PATCH 037/108] CDB_Federated_Table_Register: Handle conflict nicely --- .../CDB_FederatedServerTables.sql | 5 +++- test/CDB_FederatedServerTables.sql | 28 +++++++++++++++---- test/CDB_FederatedServerTables_expect | 8 ++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index db43d1c..4cf3afe 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -186,7 +186,10 @@ BEGIN array_to_string(carto_columns_expression || rest_of_cols, ','), src_table ); - EXCEPTION WHEN OTHERS THEN + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION 'Could not import table "%" as "%": "%" already exists', remote_table, local_name, local_name; + WHEN OTHERS THEN RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; END; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 604dcfd..62d2bc2 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -242,11 +242,29 @@ SELECT cartodb.CDB_Federated_Table_Unregister( remote_table => 'remote_geom' ); --- Try to register with invalid / NULL id --- Try to register with invalid / NULL geom_column --- Try to register with invalid / NULL webmercator_column --- Try to register the same table twice (different name) should fail --- Check that conflict is handled nicely (target view already exists) + +\echo '## Target conflict is handled nicely: Table' +CREATE TABLE localtable (a integer); +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); + +\echo '## Target conflict is handled nicely: View' +CREATE VIEW localtable2 AS Select * from localtable; +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable2'); + +DROP VIEW localtable2; +DROP TABLE localtable; -- Try permissions tricks diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 497c057..a81614b 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -53,4 +53,12 @@ ERROR: non geometry column "Does not exists" ## Registering a table: NULL webmercator_column is OK +## Target conflict is handled nicely: Table +CREATE TABLE +ERROR: Could not import table "remote_geom" as "localtable": "localtable" already exists +## Target conflict is handled nicely: View +CREATE VIEW +ERROR: Could not import table "remote_geom" as "localtable2": "localtable2" already exists +DROP VIEW +DROP TABLE D1| From e6f07d4f96bf6557f02477e2dd1adb5e4c5de879 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 17:18:49 +0100 Subject: [PATCH 038/108] Rework API to grant and revoke permissions --- scripts-available/CDB_FederatedServer.sql | 32 +++++++---------------- test/CDB_FederatedServer.sql | 21 +++++---------- test/CDB_FederatedServerListRemote.sql | 27 ++++++------------- test/CDB_FederatedServerListRemote_expect | 3 --- test/CDB_FederatedServerTables.sql | 25 +++++++++++++++++- test/CDB_FederatedServer_expect | 3 +-- 6 files changed, 50 insertions(+), 61 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 0e113a7..af96f9f 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -347,23 +347,17 @@ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- -- Grant access to a server -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, usernames text[]) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME) RETURNS void AS $$ DECLARE server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); - user_role TEXT; - username TEXT; BEGIN - FOREACH username IN ARRAY usernames - LOOP - user_role := @extschema@._CDB_User_RoleFromUsername(username); - IF (user_role IS NULL) THEN - RAISE EXCEPTION 'User role "%" does not exists', username; - END IF; - EXECUTE format('GRANT %I TO %I', server_role_name, user_role); - END loop; + IF (db_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" cannot be NULL', username; + END IF; + EXECUTE format('GRANT %I TO %I', server_role_name, db_role); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; @@ -371,23 +365,17 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- Revoke access to a server -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, usernames text[]) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME) RETURNS void AS $$ DECLARE server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); - user_role TEXT; - username TEXT; BEGIN - FOREACH username IN ARRAY usernames - LOOP - user_role := @extschema@._CDB_User_RoleFromUsername(username); - IF (user_role IS NULL) THEN - RAISE EXCEPTION 'User role "%" does not exists', username; - END IF; - EXECUTE format('REVOKE %I FROM %I', server_role_name, user_role); - END loop; + IF (db_role IS NULL) THEN + RAISE EXCEPTION 'User role "%" cannot be NULL', username; + END IF; + EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index a818f8e..287fdb0 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -138,14 +138,8 @@ SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote" or''n -- Test permissions \set QUIET on - --- We create a username following the same steps as organization members CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; -CREATE SCHEMA cdb_fs_tester AUTHORIZATION cdb_fs_tester; -SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester'); -ALTER ROLE cdb_fs_tester SET search_path TO cdb_fs_tester,cartodb,public; - \set QUIET off SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::text, config := '{ @@ -180,16 +174,16 @@ SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::te \c contrib_regression postgres \echo '## Granting access to a user works' -SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); -SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', usernames := ARRAY['cdb_fs_tester']); -SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['does not exist']); +SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); +SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', db_role := 'cdb_fs_tester'::name); +SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'does not exist'::name); \echo '## Granting access again raises a notice' -SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); +SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); \echo '## Revoking access to a user works' -SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); -SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', usernames := ARRAY['cdb_fs_tester']); +SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); +SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); \echo '## Unregistering a server with active grants works' SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote3'::text); @@ -206,7 +200,7 @@ SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::t "password": "foobarino" } }'::jsonb); -SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote4', usernames := ARRAY['cdb_fs_tester']); +SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote4', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::text); @@ -217,7 +211,6 @@ SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::te -- Cleanup \set QUIET on -DROP SCHEMA cdb_fs_tester CASCADE; REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; DROP ROLE cdb_fs_tester; DROP EXTENSION postgres_fdw; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index e230ab0..9db8a80 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -7,23 +7,14 @@ SET client_min_messages TO error; SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; --- We create a username following the same steps as organization members CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; -CREATE SCHEMA cdb_fs_tester AUTHORIZATION cdb_fs_tester; -SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester'); -ALTER ROLE cdb_fs_tester SET search_path TO cdb_fs_tester,cartodb,public; - CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; -CREATE SCHEMA cdb_fs_tester2 AUTHORIZATION cdb_fs_tester2; -SELECT cartodb.CDB_Organization_Create_Member('cdb_fs_tester2'); -ALTER ROLE cdb_fs_tester2 SET search_path TO cdb_fs_tester2,cartodb,public; -- Create database to be used as remote CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; - SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ "server": { "host": "localhost", @@ -111,13 +102,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopba \c contrib_regression postgres \echo '## Test listing of remote schemas with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \c contrib_regression postgres \echo '## Test listing of remote schemas without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \c contrib_regression postgres @@ -144,13 +135,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopbac \c contrib_regression postgres \echo '## Test listing of remote tables with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); \c contrib_regression postgres \echo '## Test listing of remote tables without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); \c contrib_regression postgres @@ -180,13 +171,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \c contrib_regression postgres \echo '## Test listing of remote columns with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); \c contrib_regression postgres \echo '## Test listing of remote columns without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); \c contrib_regression postgres @@ -211,7 +202,7 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \echo '## Test listing of remote objects with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', usernames := ARRAY['cdb_fs_tester2']); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', db_role := 'cdb_fs_tester2'::name); \c contrib_regression cdb_fs_tester2 SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); @@ -219,12 +210,11 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \c contrib_regression postgres \echo '## Test that dropping the granted user works fine (sunny day)' -DROP SCHEMA cdb_fs_tester2 CASCADE; REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; DROP ROLE cdb_fs_tester2; \echo '## Test listing of remote objects with other user still works (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', usernames := ARRAY['cdb_fs_tester']); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', db_role := 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); @@ -301,7 +291,6 @@ SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text DROP DATABASE cdb_fs_tester; -- Drop role -DROP SCHEMA cdb_fs_tester CASCADE; REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; DROP ROLE cdb_fs_tester; diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index e05af3a..e4929a7 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -1,5 +1,3 @@ - - C1| C2| ## Test listing of remote schemas without permissions before the first instantiation (rainy day) @@ -120,7 +118,6 @@ c7|character c8|USER-DEFINED You are now connected to database "contrib_regression" as user "postgres". ## Test that dropping the granted user works fine (sunny day) -DROP SCHEMA REVOKE DROP ROLE ## Test listing of remote objects with other user still works (sunny day) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 62d2bc2..50663d5 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -7,7 +7,15 @@ SET client_min_messages TO error; SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; -CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; + +-- We create a username following the same steps as organization members +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; + +CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; + +-- Create database to be used as remote CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ @@ -122,6 +130,10 @@ Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( remote_schema => 'remote_schema' ); +-- =================================================================== +-- Test input +-- =================================================================== + \echo '## Registering a table: Invalid server fails' SELECT cartodb.CDB_Federated_Table_Register( server => 'Does not exist', @@ -242,6 +254,9 @@ SELECT cartodb.CDB_Federated_Table_Unregister( remote_table => 'remote_geom' ); +-- =================================================================== +-- Test conflicts +-- =================================================================== \echo '## Target conflict is handled nicely: Table' CREATE TABLE localtable (a integer); @@ -266,6 +281,10 @@ SELECT cartodb.CDB_Federated_Table_Register( DROP VIEW localtable2; DROP TABLE localtable; +-- =================================================================== +-- Test permissions +-- =================================================================== + -- Try permissions tricks -- Try registering and accessing a table as normal user @@ -284,8 +303,12 @@ DROP TABLE localtable; SET client_min_messages TO error; \set VERBOSITY terse +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; +DROP ROLE cdb_fs_tester2; + SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); DROP DATABASE cdb_fs_tester; +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; DROP ROLE cdb_fs_tester; DROP EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index aecffa1..c411d34 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -38,7 +38,6 @@ ERROR: Server information is mandatory 8.1| 8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user") 8.3| - 9.1| You are now connected to database "contrib_regression" as user "cdb_fs_tester". ## All users are able to list servers @@ -49,7 +48,7 @@ You are now connected to database "contrib_regression" as user "postgres". ## Granting access to a user works 9.5| ERROR: Server "does not exist" does not exist -ERROR: User role "does not exist" does not exists +ERROR: role "does not exist" does not exist ## Granting access again raises a notice NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" 9.8| From e98b18fd25865d23a47e57fae7b9aa41b7ea87be Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 18:07:11 +0100 Subject: [PATCH 039/108] Add test around grant permissions and accessing registered tables --- scripts-available/CDB_FederatedServer.sql | 18 ++++++- test/CDB_FederatedServerTables.sql | 63 +++++++++++++++++++++-- test/CDB_FederatedServerTables_expect | 37 +++++++++++++ test/CDB_FederatedServer_expect | 2 +- 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index af96f9f..b385dc5 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -357,7 +357,14 @@ BEGIN IF (db_role IS NULL) THEN RAISE EXCEPTION 'User role "%" cannot be NULL', username; END IF; - EXECUTE format('GRANT %I TO %I', server_role_name, db_role); + BEGIN + EXECUTE format('GRANT %I TO %I', server_role_name, db_role); + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION 'You do not have rights to grant access on "%"', server; + WHEN OTHERS THEN + RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM; + END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; @@ -375,7 +382,14 @@ BEGIN IF (db_role IS NULL) THEN RAISE EXCEPTION 'User role "%" cannot be NULL', username; END IF; - EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role); + BEGIN + EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role); + EXCEPTION + WHEN insufficient_privilege THEN + RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server; + WHEN OTHERS THEN + RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM; + END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 50663d5..ed8dc4b 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -285,14 +285,69 @@ DROP TABLE localtable; -- Test permissions -- =================================================================== --- Try permissions tricks --- Try registering and accessing a table as normal user +\echo '## Registering tables does not work without permissions' +\c contrib_regression cdb_fs_tester +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); --- Try register with one user and reading it with other --- Try register with one user and deleting it with another +\echo '## Listing registered tables does not work without permissions' +Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +\echo '## Registering tables works with granted permissions' +\c contrib_regression postgres +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom', + local_name => 'localtable'); +\echo '## Listing registered tables works with granted permissions' +Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); + +\echo '## Selecting from a registered table with granted permissions works' +Select cartodb_id, ST_AsText(the_geom) from localtable; + +\echo '## Selecting from a registered table without permissions does not work' +\c contrib_regression cdb_fs_tester2 +Select cartodb_id, ST_AsText(the_geom) from localtable; + +\echo '## Deleting a registered table without permissions does not work' +SELECT CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); + +\echo '## Only the owner can grant permissions over the server' +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); + +\echo '## Everything works for a different user when granted permissions' +\c contrib_regression postgres +SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); +\c contrib_regression cdb_fs_tester2 +Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select cartodb_id, ST_AsText(the_geom) from localtable; + +\echo '## A different user can unregister a table' +SELECT CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom' + ); +Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); + +\echo '## Only the owner can revoke permissions over the server' +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index a81614b..4026c19 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -61,4 +61,41 @@ CREATE VIEW ERROR: Could not import table "remote_geom" as "localtable2": "localtable2" already exists DROP VIEW DROP TABLE +## Registering tables does not work without permissions +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +ERROR: Not enough permissions to access the server "loopback" +## Listing registered tables does not work without permissions +ERROR: Not enough permissions to access the server "loopback" +## Registering tables works with granted permissions +You are now connected to database "contrib_regression" as user "postgres". + +You are now connected to database "contrib_regression" as user "cdb_fs_tester". + +## Listing registered tables works with granted permissions +(remote_geom2,public.different_name) +(remote_geom,public.localtable) +## Selecting from a registered table with granted permissions works +1|POINT(1 1) +2|POINT(2 2) +## Selecting from a registered table without permissions does not work +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +ERROR: permission denied for view localtable +## Deleting a registered table without permissions does not work +ERROR: Not enough permissions to access the server "loopback" +## Only the owner can grant permissions over the server +ERROR: You do not have rights to grant access on "loopback" +## Everything works for a different user when granted permissions +You are now connected to database "contrib_regression" as user "postgres". + +You are now connected to database "contrib_regression" as user "cdb_fs_tester2". +(remote_geom2,public.different_name) +(remote_geom,public.localtable) +1|POINT(1 1) +2|POINT(2 2) +## A different user can unregister a table +NOTICE: drop cascades to view localtable + +(remote_geom2,public.different_name) +## Only the owner can revoke permissions over the server +ERROR: You do not have rights to revoke access on "loopback" D1| diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index c411d34..f21f2cc 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -48,7 +48,7 @@ You are now connected to database "contrib_regression" as user "postgres". ## Granting access to a user works 9.5| ERROR: Server "does not exist" does not exist -ERROR: role "does not exist" does not exist +ERROR: Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist ## Granting access again raises a notice NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368" 9.8| From 29e2ee670facba161ab097dd58ff8b32712a7ff3 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Nov 2019 18:29:08 +0100 Subject: [PATCH 040/108] Accept PG9.5 and PG10 permission error messages --- test/CDB_FederatedServerTables.sql | 17 ++++++++++++++++- test/CDB_FederatedServerTables_expect | 4 +++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index ed8dc4b..8baccb3 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -319,7 +319,22 @@ Select cartodb_id, ST_AsText(the_geom) from localtable; \echo '## Selecting from a registered table without permissions does not work' \c contrib_regression cdb_fs_tester2 -Select cartodb_id, ST_AsText(the_geom) from localtable; +CREATE OR REPLACE FUNCTION catch_permission_error(query text) +RETURNS bool +AS $$ +BEGIN + EXECUTE query; + RETURN FALSE; +EXCEPTION + WHEN insufficient_privilege THEN + RETURN TRUE; + WHEN OTHERS THEN + RAISE WARNING 'Exception %', sqlstate; + RETURN FALSE; +END +$$ LANGUAGE 'plpgsql'; +Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from localtable$$); +DROP FUNCTION catch_permission_error(text); \echo '## Deleting a registered table without permissions does not work' SELECT CDB_Federated_Table_Unregister( diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 4026c19..40ac9d8 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -79,7 +79,9 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". 2|POINT(2 2) ## Selecting from a registered table without permissions does not work You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -ERROR: permission denied for view localtable +CREATE FUNCTION +t +DROP FUNCTION ## Deleting a registered table without permissions does not work ERROR: Not enough permissions to access the server "loopback" ## Only the owner can grant permissions over the server From 90b58fcdec02eb6946c97144f97d414d594061cd Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 10:20:53 +0100 Subject: [PATCH 041/108] Federated Server: Test with incorrect password --- test/CDB_FederatedServerListRemote.sql | 21 +++++++++++++++++++++ test/CDB_FederatedServerListRemote_expect | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index 9db8a80..b5372f4 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -270,6 +270,27 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba -- Rerun should be ok SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5'); + +-- =================================================================== +-- Test invalid password +-- =================================================================== + +\echo '## Check error message with invalid password (rainy day)' +SELECT cartodb.CDB_Federated_Server_Register_PG(server := 'loopback_invalid'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "wrong password" + } +}'::jsonb); + +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid'); + +SELECT cartodb.CDB_Federated_Server_Unregister(server := 'loopback_invalid'::text); + -- =================================================================== -- Cleanup 2 -- =================================================================== diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index e4929a7..70b63b3 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -149,5 +149,9 @@ geo_nosrid|GEOMETRY,0 geog|Geometry,0 geom|GEOMETRY,4326 geom_wm|GEOMETRY,3857 +## Check error message with invalid password (rainy day) + +ERROR: could not connect to server "cdb_fs_loopback_invalid" + D1| D2| From 01258530ff4a9e17446afd3d18ffd3e1df139bc2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 17:04:11 +0100 Subject: [PATCH 042/108] Federated Server: Unify remote table listing - Removes DB_Federated_Server_List_Registered_Tables in favor of CDB_Federated_Server_List_Remote_Tables that now returns both registered and unregistered tables and, when they are registered, the information about the associated view --- .../CDB_FederatedServerListRemote.sql | 110 ++++++++---- .../CDB_FederatedServerTables.sql | 169 ++++++++++++------ test/CDB_FederatedServerListRemote.sql | 1 - test/CDB_FederatedServerListRemote_expect | 45 ++--- test/CDB_FederatedServerTables.sql | 32 ++-- test/CDB_FederatedServerTables_expect | 49 +++-- 6 files changed, 257 insertions(+), 149 deletions(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index d8932c0..845660f 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -30,7 +30,8 @@ BEGIN WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) AND relname = remote_table ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_table, server_internal, local_schema); END IF; -- Return the result we're interested in. Exclude toast and temp schemas @@ -49,7 +50,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- --- List the tables from a remote PG schema +-- List the names of the tables in a remote PG schema -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name) RETURNS TABLE(remote_table name) @@ -74,10 +75,10 @@ BEGIN WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) AND relname = remote_table ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_table, server_internal, local_schema); + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_table, server_internal, local_schema); END IF; - -- Return the result we're interested in -- Note: in this context, schema names are not to be quoted RETURN QUERY EXECUTE format($q$ SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name @@ -88,10 +89,10 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- --- List the columns from a remote PG table +-- List the columns in a remote PG schema -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name, remote_table name) -RETURNS TABLE(column_name name, column_type text) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name) +RETURNS TABLE(table_name name, column_name name, column_type text) AS $func$ DECLARE -- Import `columns` from the information schema @@ -112,30 +113,37 @@ BEGIN WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) AND relname = remote_col_table ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', inf_schema, remote_col_table, server_internal, local_schema); + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + inf_schema, remote_col_table, server_internal, local_schema); END IF; - -- Return the result we're interested in - -- Note: in this context, remote schema and remote table names are not to be quoted + -- Note: in this context, schema names are not to be quoted + -- We join with the geometry columns to change the type `USER-DEFINED` + -- by its appropiate geometry and srid RETURN QUERY EXECUTE format($q$ SELECT - a.column_name::name, COALESCE(b.column_type, a.data_type)::TEXT as column_type - FROM %I.%I a - LEFT JOIN @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s', '%s') b ON a.column_name = b.column_name - WHERE table_schema = '%s' AND table_name = '%s' - ORDER BY column_name$q$, + a.table_name::name, + a.column_name::name, + COALESCE(b.column_type, a.data_type)::TEXT as column_type + FROM + %I.%I a + LEFT JOIN + @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b + ON a.table_name = b.table_name AND a.column_name = b.column_name + WHERE table_schema = '%s' + ORDER BY a.table_name, a.column_name $q$, local_schema, remote_col_table, - server_internal, remote_schema, remote_table, - remote_schema, remote_table); + server_internal, remote_schema, + remote_schema); END $func$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- --- List the columns from a remote PG table +-- List the geometry columns in a remote PG schema -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, remote_table name, postgis_schema name DEFAULT 'public') -RETURNS TABLE(column_name name, column_type text) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public') +RETURNS TABLE(table_name name, column_name name, column_type text) AS $func$ DECLARE -- Import `geometry_columns` and `geography_columns` from the postgis schema @@ -159,21 +167,22 @@ BEGIN BEGIN -- Note: We return both the type and srid as the type RETURN QUERY EXECUTE format($q$ - SELECT f_geometry_column::NAME as column_name, - type::TEXT || ',' || srid::TEXT as column_type - FROM - ( - SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I - ) _geo_views - WHERE - f_table_schema = '%s' AND - f_table_name = '%s' + SELECT + f_table_name::NAME as table_name, + f_geometry_column::NAME as column_name, + type::TEXT || ',' || srid::TEXT as column_type + FROM + ( + SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I + ) _geo_views + WHERE f_table_schema = '%s' $q$, local_schema, remote_geometry_view, local_schema, remote_geography_view, - remote_schema, remote_table); + remote_schema); EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Could not find Postgis installation in the remote "%" schema', postgis_schema; + RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"', + postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal); RETURN; END; END @@ -207,9 +216,18 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List remote tables in a federated server that the current user has access to. +-- For registered tables it returns also the associated configuration -- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT) -RETURNS TABLE(remote_table name) +RETURNS TABLE( + registered boolean, + remote_table TEXT, + local_qualified_name TEXT, + id_column_name TEXT, + geom_column_name TEXT, + webmercator_column_name TEXT, + columns JSON + ) AS $$ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); @@ -217,7 +235,29 @@ DECLARE BEGIN CASE server_type WHEN 'postgres_fdw' THEN - RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema); + RETURN QUERY + SELECT + coalesce(registered_tables.registered, false)::boolean as registered, + foreign_tables.remote_table::text as remote_table, + registered_tables.local_qualified_name as local_qualified_name, + registered_tables.id_column_name as id_column_name, + registered_tables.geom_column_name as geom_column_name, + registered_tables.webmercator_column_name as webmercator_column_name, + remote_columns.columns as columns + FROM + @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables + LEFT JOIN + @extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables + ON foreign_tables.remote_table = registered_tables.remote_table + LEFT JOIN + ( -- Extract and group columns with their remote table + SELECT table_name, + json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns + FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) + GROUP BY table_name + ) remote_columns + ON foreign_tables.remote_table = remote_columns.table_name + ORDER BY foreign_tables.remote_table; ELSE RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; END CASE; @@ -232,7 +272,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( server TEXT, remote_schema TEXT, remote_table TEXT) -RETURNS TABLE(column_name name, column_type text) +RETURNS TABLE(column_n name, column_t text) AS $$ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); @@ -244,7 +284,7 @@ BEGIN CASE server_type WHEN 'postgres_fdw' THEN - RETURN QUERY SELECT * FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema, remote_table); + RETURN QUERY SELECT column_name, column_type FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) where table_name = remote_table; ELSE RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; END CASE; diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 4cf3afe..11497a3 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -18,7 +18,7 @@ BEGIN RETURN FOUND; END $$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; -- -- Checks if a column is of geometry type @@ -34,7 +34,7 @@ BEGIN RETURN FOUND; END $$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE; -- -- Returns the name of all the columns from a table @@ -55,7 +55,108 @@ AS $$ WHERE c.oid = input_table::oid ) ORDER BY a.attnum; -$$ LANGUAGE SQL; +$$ LANGUAGE SQL STABLE PARALLEL UNSAFE; + +-- +-- Returns the id column from a view definition +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + ) + SELECT split_part(col_def[array_length(col_def, 1) - 2], '.', 2) + FROM column_definitions where col_def[array_length(col_def, 1)] = 'cartodb_id,' + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Returns the geom column from a view definition +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + ) + SELECT trim(trailing ',' FROM split_part( + CASE WHEN col_def[array_length(col_def, 1) - 2] = '4326)' THEN col_def[array_length(col_def, 1) - 3] + ELSE col_def[array_length(col_def, 1) - 2] + END, '.', 2)) + FROM column_definitions + WHERE col_def[array_length(col_def, 1)] = 'the_geom,' + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +-- +-- Returns the webmercatorcolumn from a view definition +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT) +RETURNS TEXT +AS $$ + WITH column_definitions AS + ( + SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + ) + SELECT trim(trailing ',' FROM split_part( + CASE WHEN col_def[array_length(col_def, 1) - 2] = '3857)' THEN col_def[array_length(col_def, 1) - 3] + ELSE col_def[array_length(col_def, 1) - 2] + END, '.', 2)) + FROM column_definitions + WHERE col_def[array_length(col_def, 1)] = 'the_geom_webmercator,' + LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +-- +-- List all registered tables in a server + schema +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables( + server_internal NAME, + remote_schema TEXT + ) +RETURNS TABLE( + registered boolean, + remote_table TEXT, + local_qualified_name TEXT, + id_column_name TEXT, + geom_column_name TEXT, + webmercator_column_name TEXT + ) +AS $$ +DECLARE + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); +BEGIN + RETURN QUERY SELECT + true as registered, + source_table::text as remote_table, + format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name, + @extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name, + @extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name, + @extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name + FROM + ( + SELECT DISTINCT + dependent_ns.nspname as dependent_schema, + dependent_view.relname as dependent_view, + source_table.relname as source_table, + pg_get_viewdef(dependent_view.oid) as view_definition + FROM pg_depend + JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid + JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid + JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid + JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace + JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace + WHERE + source_ns.nspname = local_schema + ORDER BY 1,2 + ) _aux; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -------------------------------------------------------------------------------- @@ -119,15 +220,22 @@ BEGIN -- Import the foreign table -- Drop the old view / table if there was one - IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table); END IF; - EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', remote_schema, remote_table, server_internal, local_schema); - src_table := format('%I.%I', local_schema, remote_table); - - --- Grant SELECT to fdw role (TODO: Re-enable if needed) - --- EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); + BEGIN + EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', + remote_schema, remote_table, server_internal, local_schema); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import schema "%" of server "%"', remote_schema, server; + END; + BEGIN + src_table := format('%I.%I', local_schema, remote_table); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server; + END; + -- Check id_column is numeric IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN RAISE EXCEPTION 'non integer id_column "%"', id_column; @@ -155,7 +263,7 @@ BEGIN geom_expression := format('t.%I AS the_geom', geom_column); ELSE -- It needs an ST_Transform to 4326 - geom_expression := format('@postgisschema@.ST_Transform(t.%I, 4326) AS the_geom', geom_column); + geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column); END IF; IF webmercator_column IS NULL @@ -166,7 +274,7 @@ BEGIN webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column); ELSE -- It needs an ST_Transform to 3857 - webmercator_expression := format('@postgisschema@.ST_Transform(t.%I, 3857) AS the_geom_webmercator', webmercator_column); + webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column); END IF; -- CARTO columns expressions @@ -192,9 +300,6 @@ BEGIN WHEN OTHERS THEN RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; END; - - -- TODO: Handle this Grant perms to the view - -- EXECUTE format('GRANT SELECT ON %I TO %s', table_name, fdw_objects_name); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; @@ -217,39 +322,3 @@ BEGIN END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - --- --- List all registered tables in a server + schema --- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Registered_Tables( - server TEXT, - remote_schema TEXT - ) -RETURNS TABLE(remote_name TEXT, local_name TEXT) -AS $$ -DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); - local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); -BEGIN - RETURN QUERY SELECT - source_table::text as remote_table, - format('%I.%I', dependent_schema, dependent_view)::text as local_view - FROM - ( - SELECT DISTINCT - dependent_ns.nspname as dependent_schema, - dependent_view.relname as dependent_view, - source_table.relname as source_table - FROM pg_depend - JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid - JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid - JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid - JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace - JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace - WHERE - source_ns.nspname = local_schema - ORDER BY 1,2 - ) _aux; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index b5372f4..9c937e1 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -4,7 +4,6 @@ \set QUIET on SET client_min_messages TO error; \set VERBOSITY terse -SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; diff --git a/test/CDB_FederatedServerListRemote_expect b/test/CDB_FederatedServerListRemote_expect index 70b63b3..244203c 100644 --- a/test/CDB_FederatedServerListRemote_expect +++ b/test/CDB_FederatedServerListRemote_expect @@ -31,10 +31,11 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". ERROR: Not enough permissions to access the server "loopback" You are now connected to database "contrib_regression" as user "postgres". ## Test listing of remote tables (sunny day) -T 1 -T 2 -T 3 -T 4 +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] ## Test listing of remote tables without permissions after the first instantiation (rainy day) You are now connected to database "contrib_regression" as user "cdb_fs_tester". ERROR: Not enough permissions to access the server "loopback" @@ -42,10 +43,11 @@ You are now connected to database "contrib_regression" as user "postgres". ## Test listing of remote tables with permissions (sunny day) You are now connected to database "contrib_regression" as user "cdb_fs_tester". -T 1 -T 2 -T 3 -T 4 +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] You are now connected to database "contrib_regression" as user "postgres". ## Test listing of remote tables without permissions after revoking access (rainy day) @@ -60,7 +62,7 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". ERROR: Not enough permissions to access the server "loopback" You are now connected to database "contrib_regression" as user "postgres". ## Test listing of remote columns (sunny day) -NOTICE: Could not find Postgis installation in the remote "public" schema +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" C 1|integer c2|integer c3|text @@ -76,7 +78,7 @@ You are now connected to database "contrib_regression" as user "postgres". ## Test listing of remote columns with permissions (sunny day) You are now connected to database "contrib_regression" as user "cdb_fs_tester". -NOTICE: Could not find Postgis installation in the remote "public" schema +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" C 1|integer c2|integer c3|text @@ -95,6 +97,7 @@ You are now connected to database "contrib_regression" as user "postgres". ERROR: Server "Does Not Exist" does not exist ## Test listing of remote columns (rainy day): Remote schema does not exist ## Test listing of remote columns (rainy day): Remote table does not exist +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback" ## Test listing of remote columns (rainy day): Remote table is NULL ERROR: Remote table name cannot be NULL ## Test listing of remote objects with permissions (sunny day) @@ -103,11 +106,12 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester2". S 1 information_schema public -T 1 -T 2 -T 3 -T 4 -NOTICE: Could not find Postgis installation in the remote "public" schema +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" C 1|integer c2|integer c3|text @@ -126,11 +130,12 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". S 1 information_schema public -T 1 -T 2 -T 3 -T 4 -NOTICE: Could not find Postgis installation in the remote "public" schema +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" +f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}] +f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}] +f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}] +INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2" C 1|integer c2|integer c3|text diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 8baccb3..4200a4f 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -5,7 +5,6 @@ SET client_min_messages TO error; \set VERBOSITY terse -SET SESSION AUTHORIZATION postgres; CREATE EXTENSION postgres_fdw; -- We create a username following the same steps as organization members @@ -72,10 +71,8 @@ SELECT 'R1', cartodb.CDB_Federated_Table_Register( geom_column => 'geom' ); -SELECT 'V1', pg_get_viewdef('remote_geom'); SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; - -Select 'list_remotes1', CDB_Federated_Server_List_Registered_Tables( +Select * FROM CDB_Federated_Server_List_Remote_Tables( server => 'loopback', remote_schema => 'remote_schema' ); @@ -91,29 +88,28 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( local_name => 'myFullTable' ); -SELECT 'V2', pg_get_viewdef('"myFullTable"'); SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTable"; - -Select 'list_remotes2', CDB_Federated_Server_List_Registered_Tables( +Select * FROM CDB_Federated_Server_List_Remote_Tables( server => 'loopback', remote_schema => 'remote_schema' ); - \echo '## Re-registering a table works' SELECT 'R3', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom2', id_column => 'id', - geom_column => 'geom', + -- Switch geom and geom_column on purpose to force ST_Transform to be used + geom_column => 'geom_mercator', + webmercator_column => 'geom', local_name => 'different_name' ); -- The old view should dissapear -SELECT 'S3_old', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTable"; +SELECT 'S3_old', cartodb_id, another_field FROM "myFullTable"; -- And the new appear -SELECT 'S3_new', cartodb_id, ST_AsText(the_geom), another_field FROM different_name; +SELECT 'S3_new', cartodb_id, another_field FROM different_name; \echo '## Unregistering works' -- Deregistering the first table @@ -125,7 +121,7 @@ SELECT 'U1', CDB_Federated_Table_Unregister( -- Selecting from the created view should fail now SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; -Select 'list_remotes3', CDB_Federated_Server_List_Registered_Tables( +Select * FROM CDB_Federated_Server_List_Remote_Tables( server => 'loopback', remote_schema => 'remote_schema' ); @@ -296,8 +292,8 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'geom', local_name => 'localtable'); -\echo '## Listing registered tables does not work without permissions' -Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +\echo '## Listing remote tables does not work without permissions' +Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Registering tables works with granted permissions' \c contrib_regression postgres @@ -311,8 +307,8 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'geom', local_name => 'localtable'); -\echo '## Listing registered tables works with granted permissions' -Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +\echo '## Listing remote tables works with granted permissions' +Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Selecting from a registered table with granted permissions works' Select cartodb_id, ST_AsText(the_geom) from localtable; @@ -350,7 +346,7 @@ SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role : \c contrib_regression postgres SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); \c contrib_regression cdb_fs_tester2 -Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); Select cartodb_id, ST_AsText(the_geom) from localtable; \echo '## A different user can unregister a table' @@ -359,7 +355,7 @@ SELECT CDB_Federated_Table_Unregister( remote_schema => 'remote_schema', remote_table => 'remote_geom' ); -Select CDB_Federated_Server_List_Registered_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Only the owner can revoke permissions over the server' SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 40ac9d8..8267a72 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -1,42 +1,37 @@ C1| ## Registering an existing table works R1| -V1| SELECT t.id AS cartodb_id, - t.geom AS the_geom, - st_transform(t.geom, 3857) AS the_geom_webmercator, - t.another_field - FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom t; S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 -list_remotes1|(remote_geom,public.remote_geom) +t|remote_geom|public.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering another existing table works R2| -V2| SELECT t.id AS cartodb_id, - t.geom AS the_geom, - t.geom_mercator AS the_geom_webmercator, - t.another_field - FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom2 t; S2|3|POINT(3 3)|patata -list_remotes2|(remote_geom2,"public.""myFullTable""") -list_remotes2|(remote_geom,public.remote_geom) +t|remote_geom|public.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|public."myFullTable"|id|geom|geom_mercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Re-registering a table works R3| -ERROR: relation "myFullTable" does not exist at character 70 -S3_new|3|POINT(3 3)|patata +ERROR: relation "myFullTable" does not exist at character 49 +S3_new|3|patata ## Unregistering works U1| ERROR: relation "remote_geom" does not exist at character 71 -list_remotes3|(remote_geom2,public.different_name) +f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering a table: Invalid server fails ERROR: Server "Does not exist" does not exist ## Registering a table: NULL server fails ERROR: Server name cannot be NULL ## Registering a table: Invalid schema fails -ERROR: schema "Does not exist" is not present on foreign server "cdb_fs_loopback" +ERROR: Could not import schema "Does not exist" of server "loopback" ## Registering a table: NULL schema fails ERROR: Schema name cannot be NULL ## Registering a table: Invalid table fails -ERROR: relation "cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.Does not exist" does not exist +ERROR: Could not import table "remote_schema.Does not exist" of server "loopback" ## Registering a table: NULL table fails ERROR: Remote table name cannot be NULL ## Registering a table: Invalid id fails @@ -64,16 +59,17 @@ DROP TABLE ## Registering tables does not work without permissions You are now connected to database "contrib_regression" as user "cdb_fs_tester". ERROR: Not enough permissions to access the server "loopback" -## Listing registered tables does not work without permissions +## Listing remote tables does not work without permissions ERROR: Not enough permissions to access the server "loopback" ## Registering tables works with granted permissions You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester". -## Listing registered tables works with granted permissions -(remote_geom2,public.different_name) -(remote_geom,public.localtable) +## Listing remote tables works with granted permissions +t|remote_geom|public.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Selecting from a registered table with granted permissions works 1|POINT(1 1) 2|POINT(2 2) @@ -90,14 +86,17 @@ ERROR: You do not have rights to grant access on "loopback" You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -(remote_geom2,public.different_name) -(remote_geom,public.localtable) +t|remote_geom|public.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] 1|POINT(1 1) 2|POINT(2 2) ## A different user can unregister a table NOTICE: drop cascades to view localtable -(remote_geom2,public.different_name) +f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Only the owner can revoke permissions over the server ERROR: You do not have rights to revoke access on "loopback" D1| From 899a12fc5305d50a0872e3766df0a0cdcb6ae8b8 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 17:12:40 +0100 Subject: [PATCH 043/108] FederatedServer: Use NAME for internal types --- scripts-available/CDB_FederatedServer.sql | 20 +++++++++---------- .../CDB_FederatedServerListRemote.sql | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index b385dc5..4d9f5d1 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -59,7 +59,7 @@ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- -- Produce a valid name for a schema generated for the Federated Server -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name TEXT, schema_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT) RETURNS NAME AS $$ DECLARE @@ -77,7 +77,7 @@ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- Produce a valid name for a role generated for the Federated Server -- This needs to include the current database in its hash to avoid collisions in clusters with more than one database -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME) RETURNS NAME AS $$ DECLARE @@ -93,12 +93,12 @@ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- Creates (if not exist) a schema to place the objects for a remote schema -- The schema is with the same AUTHORIZATION as the server -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name TEXT, schema_name TEXT) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT) RETURNS NAME AS $$ DECLARE - schema_name text := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); - role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); + schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); BEGIN -- By changing the local role to the owner of the server we have an -- easy way to check for permissions and keep all objects under the same owner @@ -125,13 +125,13 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- Returns the type of a server by internal name -- Currently all of them should be postgres_fdw -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(remote_server name) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME) RETURNS name AS $$ SELECT f.fdwname FROM pg_foreign_server s JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid - WHERE s.srvname = remote_server; + WHERE s.srvname = internal_server_name; $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; @@ -287,7 +287,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TE RETURNS void AS $$ DECLARE - server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN SET client_min_messages = ERROR; @@ -351,7 +351,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server RETURNS void AS $$ DECLARE - server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN IF (db_role IS NULL) THEN @@ -376,7 +376,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server RETURNS void AS $$ DECLARE - server_internal text := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN IF (db_role IS NULL) THEN diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 845660f..115f360 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -22,7 +22,7 @@ DECLARE inf_schema name := 'information_schema'; remote_table name := 'schemata'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); - role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN -- Import the foreign schemata table IF NOT EXISTS ( From a91f14371ab0278feba7d4719cb800b88dd31521 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 17:25:10 +0100 Subject: [PATCH 044/108] Use always the modern notation for named parameters --- scripts-available/CDB_FederatedServer.sql | 10 ++-- .../CDB_FederatedServerListRemote.sql | 6 +- .../CDB_FederatedServerTables.sql | 4 +- test/CDB_FederatedServer.sql | 60 +++++++++---------- test/CDB_FederatedServerListRemote.sql | 28 ++++----- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 4d9f5d1..32ec32a 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -215,7 +215,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server T RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config)); role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); row record; @@ -287,13 +287,13 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TE RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN SET client_min_messages = ERROR; BEGIN EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); - EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); + EXECUTE FORMAT ('DROP OWNED BY %I', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; @@ -351,7 +351,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN IF (db_role IS NULL) THEN @@ -376,7 +376,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN IF (db_role IS NULL) THEN diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index 115f360..f5a0aaf 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -201,7 +201,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas( RETURNS TABLE(remote_schema name) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type @@ -230,7 +230,7 @@ RETURNS TABLE( ) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type @@ -275,7 +275,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns( RETURNS TABLE(column_n name, column_t text) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN IF remote_table IS NULL THEN diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 11497a3..c54cf06 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -196,7 +196,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); src_table REGCLASS; @@ -315,7 +315,7 @@ CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister( RETURNS void AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); BEGIN EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table); diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 287fdb0..1f418c2 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -10,10 +10,10 @@ CREATE EXTENSION postgres_fdw; SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers(); \echo '## List non-existent server shows nothing' -SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server := 'doesNotExist'); +SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server => 'doesNotExist'); \echo '## Create and list a server works' -SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::text, config := '{ +SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -26,7 +26,7 @@ SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote'::tex SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers(); \echo '## Create and list a second server works' -SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ +SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ "server": { "dbname": "fdw_target", "host": "localhost", @@ -44,11 +44,11 @@ SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers(); \echo '## List server by name works' -SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server := 'myRemote'); +SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server => 'myRemote'); \echo '## Re-register a second server works' -SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::text, config := '{ +SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{ "server": { "dbname": "fdw_target", "host": "localhost", @@ -66,24 +66,24 @@ SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote2'::te SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers(); \echo '## Unregister server 1 works' -SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote'::text); +SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote'::text); SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers(); \echo '## Unregistering a server that does not exist fails' -SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server := 'doesNotExist'::text); +SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server => 'doesNotExist'::text); \echo '## Unregister the second server works' -SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote2'::text); +SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote2'::text); SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers(); \echo '## Create a server with NULL name fails' -SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server := NULL::text, config := '{ "server": {}, "credentials" : {}}'); +SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server => NULL::text, config => '{ "server": {}, "credentials" : {}}'); \echo '## Create a server with NULL config fails' -SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := NULL::jsonb); +SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => NULL::jsonb); \echo '## Create a server with empty config fails' -SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{}'); +SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{}'); \echo '## Create a server without credentials fails' -SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ +SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ "server": { "dbname": "fdw_target", "host": "localhost", @@ -95,7 +95,7 @@ SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, } }'::jsonb); \echo '## Create a server with empty credentials works' -SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ +SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ "server": { "dbname": "fdw_target", "host": "localhost", @@ -108,9 +108,9 @@ SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, "credentials": { } }'::jsonb); SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers(); -SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server := 'empty'::text); +SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server => 'empty'::text); \echo '## Create a server without options fails' -SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, config := '{ +SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{ "credentials": { "username": "other_remote_user", "password": "foobarino" @@ -118,7 +118,7 @@ SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server := 'empty'::text, }'::jsonb); \echo '## Create a server with special characters works' -SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or''not'::text, config := '{ +SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote" or''not'::text, config => '{ "server": { "dbname": "fdw target", "host": "localhost", @@ -134,7 +134,7 @@ SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote" or'' } }'::jsonb); SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers(); -SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote" or''not'::text); +SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote" or''not'::text); -- Test permissions \set QUIET on @@ -142,7 +142,7 @@ CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; \set QUIET off -SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::text, config := '{ +SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote3'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -159,7 +159,7 @@ SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote3'::te SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers(); \echo '## Only superadmins can create servers' -SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ +SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -174,23 +174,23 @@ SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::te \c contrib_regression postgres \echo '## Granting access to a user works' -SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); -SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server := 'does not exist', db_role := 'cdb_fs_tester'::name); -SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'does not exist'::name); +SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); +SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name); +SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name); \echo '## Granting access again raises a notice' -SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); +SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); \echo '## Revoking access to a user works' -SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); -SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote3', db_role := 'cdb_fs_tester'::name); +SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); +SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); \echo '## Unregistering a server with active grants works' -SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote3'::text); +SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote3'::text); \echo '## A user with granted access can not drop a server' -SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::text, config := '{ +SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -200,13 +200,13 @@ SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server := 'myRemote4'::t "password": "foobarino" } }'::jsonb); -SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server := 'myRemote4', db_role := 'cdb_fs_tester'::name); +SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote4', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester -SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::text); +SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); \c contrib_regression postgres -SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server := 'myRemote4'::text); +SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text); -- Cleanup diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index 9c937e1..8114673 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -14,7 +14,7 @@ GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2; -- Create database to be used as remote CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; -SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -25,7 +25,7 @@ SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text } }'::jsonb); -SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback2'::text, config := '{ +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback2'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -101,13 +101,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopba \c contrib_regression postgres \echo '## Test listing of remote schemas with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \c contrib_regression postgres \echo '## Test listing of remote schemas without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \c contrib_regression postgres @@ -134,13 +134,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopbac \c contrib_regression postgres \echo '## Test listing of remote tables with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); \c contrib_regression postgres \echo '## Test listing of remote tables without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); \c contrib_regression postgres @@ -170,13 +170,13 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \c contrib_regression postgres \echo '## Test listing of remote columns with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); \c contrib_regression postgres \echo '## Test listing of remote columns without permissions after revoking access (rainy day)' -SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1'); \c contrib_regression postgres @@ -201,7 +201,7 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba \echo '## Test listing of remote objects with permissions (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', db_role := 'cdb_fs_tester2'::name); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester2'::name); \c contrib_regression cdb_fs_tester2 SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); @@ -213,7 +213,7 @@ REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2; DROP ROLE cdb_fs_tester2; \echo '## Test listing of remote objects with other user still works (sunny day)' -SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback2', db_role := 'cdb_fs_tester'::name); +SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester'::name); \c contrib_regression cdb_fs_tester SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2'); SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1'); @@ -275,7 +275,7 @@ SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopba -- =================================================================== \echo '## Check error message with invalid password (rainy day)' -SELECT cartodb.CDB_Federated_Server_Register_PG(server := 'loopback_invalid'::text, config := '{ +SELECT cartodb.CDB_Federated_Server_Register_PG(server => 'loopback_invalid'::text, config => '{ "server": { "host": "localhost", "port": @@PGPORT@@ @@ -288,7 +288,7 @@ SELECT cartodb.CDB_Federated_Server_Register_PG(server := 'loopback_invalid'::te SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid'); -SELECT cartodb.CDB_Federated_Server_Unregister(server := 'loopback_invalid'::text); +SELECT cartodb.CDB_Federated_Server_Unregister(server => 'loopback_invalid'::text); -- =================================================================== -- Cleanup 2 @@ -305,8 +305,8 @@ DROP SCHEMA "S 1"; SET client_min_messages TO error; \set VERBOSITY terse -SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); -SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'loopback2'::text); DROP DATABASE cdb_fs_tester; From 3cde55ed3a5fadb80576b24f3c3cc681ea02a675 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 17:32:48 +0100 Subject: [PATCH 045/108] Enforce output order in CDB_Federated_Server_List_Remote_Columns --- scripts-available/CDB_FederatedServerListRemote.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index f5a0aaf..a7085ed 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -284,7 +284,13 @@ BEGIN CASE server_type WHEN 'postgres_fdw' THEN - RETURN QUERY SELECT column_name, column_type FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) where table_name = remote_table; + RETURN QUERY + SELECT + column_name, + column_type + FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema) + WHERE table_name = remote_table + ORDER BY column_name; ELSE RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; END CASE; From 82d8f7e1aa6a968f80f994acdcd7bbc7f707e14a Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 18:26:37 +0100 Subject: [PATCH 046/108] Deprecate FDW user setup ;__( --- scripts-available/CDB_ForeignTable.sql | 179 ++---------------------- scripts-available/CDB_Organizations.sql | 26 +--- test/extension/test.sh | 62 -------- 3 files changed, 12 insertions(+), 255 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index c86f5ef..b9449de 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -139,176 +139,6 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; --- Produce a valid DB name for objects created for the user FDW's -CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME) -RETURNS NAME AS $$ - -- Note on input we use %s and on output we use %I, in order to - -- avoid double escaping - SELECT format('cdb_fdw_%s', fdw_input_name)::NAME; -$$ -LANGUAGE sql IMMUTABLE PARALLEL SAFE; - --- A function to set up a user-defined foreign data server --- It does not read from CDB_Conf. --- Only superuser roles can invoke it successfully --- --- Sample call: --- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{ --- "server": { --- "extensions": "postgis", --- "dbname": "testdb", --- "host": "myhostname.us-east-2.rds.amazonaws.com", --- "port": "5432" --- }, --- "user_mapping": { --- "user": "fdw_user", --- "password": "secret" --- } --- }'); --- --- Underneath it will: --- * Set up postgresql_fdw --- * Create a server with the name 'cdb_fdw_amazon' --- * Create a role called 'cdb_fdw_amazon' to manage access --- * Create a user mapping with that role 'cdb_fdw_amazon' --- * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign --- tables over there --- --- It is the responsibility of the superuser to grant that role to either: --- * Nobody --- * Specific roles: GRANT amazon TO role_name; --- * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon'); --- * The publicuser: GRANT cdb_fdw_amazon TO publicuser; -CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json) -RETURNS void AS $$ -DECLARE - row record; - option record; - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); -BEGIN - -- TODO: refactor with original function - -- This function tries to be as idempotent as possible, by not creating anything more than once - -- (not even using IF NOT EXIST to avoid throwing warnings) - IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN - CREATE EXTENSION postgres_fdw; - RAISE NOTICE 'Created postgres_fdw EXTENSION'; - END IF; - -- Create FDW first if it does not exist - IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name) - THEN - EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name); - RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name; - END IF; - - -- Set FDW settings - FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p - LOOP - IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key) - THEN - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value); - ELSE - EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value); - END IF; - END LOOP; - - -- Create specific role for this - IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN - EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name); - RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name; - END IF; - - -- Transfer ownership of the server to the fdw role - EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name); - - -- Create user mapping - -- NOTE: we use a PUBLIC user mapping but control access to the SERVER - -- so that we don't need to create a mapping for every user nor store credentials elsewhere - IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN - EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name); - RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name; - END IF; - - -- Update user mapping settings - FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP - IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value); - ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value); - END IF; - END LOOP; - - -- Grant usage on the wrapper and server to the fdw role - EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name); - RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name; - EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name); - RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name; - - -- Create schema if it does not exist. - IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN - EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name); - RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name; - END IF; - - -- Give the fdw role ownership over the schema - EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name); - RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name; - - -- TODO: Bring here the remote cdb_tablemetadata -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - --- A function to drop a user-defined foreign server and all related objects --- It does not read from CDB_Conf --- It must be executed with a superuser role to succeed --- --- Sample call: --- SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon') --- --- Note: if there's any dependent object (i.e. foreign table) this call will fail -CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false) -RETURNS void AS $$ -DECLARE - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); - cascade_clause NAME; -BEGIN - CASE force - WHEN true THEN - cascade_clause := 'CASCADE'; - ELSE - cascade_clause := 'RESTRICT'; - END CASE; - - EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Dropped schema %', fdw_objects_name; - EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name); - RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name; - EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Dropped foreign server %', fdw_objects_name; - EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause); - RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name; - EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name); - RAISE NOTICE 'Dropped role %', fdw_objects_name; -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - --- Set up a user foreign table --- E.g: --- SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable'); --- SELECT * FROM amazon.my_table; -CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME) -RETURNS void AS $$ -DECLARE - fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name); -BEGIN - EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name); - --- Grant SELECT to fdw role - EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); -END -$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; - - CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid) RETURNS TEXT AS $$ SELECT option_value FROM pg_options_to_table(( @@ -374,3 +204,12 @@ RETURNS timestamptz AS $$ LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid ) SELECT max(updated_at) FROM t_updated_at; $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + + +-------------------------------------------------------------------------------- +-- Deprecated +-------------------------------------------------------------------------------- +DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name); +DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json); +DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean); +DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name); diff --git a/scripts-available/CDB_Organizations.sql b/scripts-available/CDB_Organizations.sql index c532ed5..94e228e 100644 --- a/scripts-available/CDB_Organizations.sql +++ b/scripts-available/CDB_Organizations.sql @@ -172,27 +172,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -------------------------------------------------------------------------------- --- Role management +-- Deprecated -------------------------------------------------------------------------------- -CREATE OR REPLACE -FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name) -RETURNS VOID AS $$ -DECLARE - org_role TEXT; -BEGIN - org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); - EXECUTE format('GRANT %I TO %I', role_name, org_role); -END -$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - - -CREATE OR REPLACE -FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name) -RETURNS VOID AS $$ -DECLARE - org_role TEXT; -BEGIN - org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name(); - EXECUTE format('REVOKE %I FROM %I', role_name, org_role); -END -$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name); +DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name); diff --git a/test/extension/test.sh b/test/extension/test.sh index faf3d93..aab19f3 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -598,68 +598,6 @@ test_extension|public|"local-table-with-dashes"' sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;" sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't' - - # Check user-defined FDW's - # Set up a user foreign server - read -d '' ufdw_config <<- EOF -{ - "server": { - "extensions": "postgis", - "dbname": "fdw_target", - "host": "localhost", - "port": ${PGPORT:-5432} - }, - "user_mapping": { - "user": "fdw_user", - "password": "foobarino" - } -} -EOF - sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');" - - # Grant a user access to that FDW, and to grant to others - sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;' - - # Set up a user foreign table - sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');" - - # Check that the table can be accessed by the owner/creator - sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;' - sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - - # Check that a role with no permissions cannot use the FDW to access a remote table - sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails - - # Check that the table can be accessed by some other user by granting the role - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;' - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;' - - # Check that the table can be accessed by org members - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');" - sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');" - - # By default publicuser cannot access the FDW - sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails - sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted - sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42 - sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;' - - # If there are dependent objects, we cannot drop the foreign server - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails - sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;' - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" - - # But if there are, we can set the force flag to true to drop everything (defaults to false) - sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');" - sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;' - sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');" - sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)" - - # Teardown DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;' From c64c61781e123779066d8e19d899f43d5e0cb028 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 18:26:58 +0100 Subject: [PATCH 047/108] Make tests path insensitive --- test/CDB_FederatedServerTables.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 4200a4f..328a05b 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -113,7 +113,7 @@ SELECT 'S3_new', cartodb_id, another_field FROM different_name; \echo '## Unregistering works' -- Deregistering the first table -SELECT 'U1', CDB_Federated_Table_Unregister( +SELECT 'U1', cartodb.CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom' @@ -293,7 +293,7 @@ SELECT cartodb.CDB_Federated_Table_Register( local_name => 'localtable'); \echo '## Listing remote tables does not work without permissions' -Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Registering tables works with granted permissions' \c contrib_regression postgres @@ -308,7 +308,7 @@ SELECT cartodb.CDB_Federated_Table_Register( local_name => 'localtable'); \echo '## Listing remote tables works with granted permissions' -Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Selecting from a registered table with granted permissions works' Select cartodb_id, ST_AsText(the_geom) from localtable; @@ -333,7 +333,7 @@ Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from loca DROP FUNCTION catch_permission_error(text); \echo '## Deleting a registered table without permissions does not work' -SELECT CDB_Federated_Table_Unregister( +SELECT cartodb.CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom' @@ -346,16 +346,16 @@ SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role : \c contrib_regression postgres SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); \c contrib_regression cdb_fs_tester2 -Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); Select cartodb_id, ST_AsText(the_geom) from localtable; \echo '## A different user can unregister a table' -SELECT CDB_Federated_Table_Unregister( +SELECT cartodb.CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom' ); -Select * FROM CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Only the owner can revoke permissions over the server' SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name); From d5b39535689107311b6612d88a66349957d9b6dd Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 6 Nov 2019 18:34:15 +0100 Subject: [PATCH 048/108] Regress: Remove output that depends on PROJ release --- test/CDB_CartodbfyTableTest.sql | 4 ++-- test/CDB_CartodbfyTableTest_expect | 4 ++-- test/CDB_SyncTableTest.sql | 2 +- test/CDB_SyncTableTest_expect | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/CDB_CartodbfyTableTest.sql b/test/CDB_CartodbfyTableTest.sql index 3e5658e..1963eaf 100644 --- a/test/CDB_CartodbfyTableTest.sql +++ b/test/CDB_CartodbfyTableTest.sql @@ -134,9 +134,9 @@ DROP TABLE t; -- table with single non-geometrical column CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT * FROM t; +SELECT cartodb_id, the_geom, description FROM t; SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence'); -SELECT * FROM t; +SELECT cartodb_id, the_geom, description FROM t; DROP TABLE t; -- table with existing srid-unconstrained (but type-constrained) the_geom diff --git a/test/CDB_CartodbfyTableTest_expect b/test/CDB_CartodbfyTableTest_expect index 9939598..c5030a0 100644 --- a/test/CDB_CartodbfyTableTest_expect +++ b/test/CDB_CartodbfyTableTest_expect @@ -7,9 +7,9 @@ single non-geometrical column cartodbfied fine DROP TABLE SELECT 1 check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence check function idempotence cartodbfied fine -1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence +1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence DROP TABLE SELECT 1 srid-unconstrained the_geom cartodbfied fine diff --git a/test/CDB_SyncTableTest.sql b/test/CDB_SyncTableTest.sql index fc2711e..5b945ae 100644 --- a/test/CDB_SyncTableTest.sql +++ b/test/CDB_SyncTableTest.sql @@ -51,7 +51,7 @@ UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecodin SET client_min_messages TO notice; SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}'); SELECT * FROM test_sync_source ORDER BY cartodb_id; -SELECT * FROM test_sync_dest ORDER BY cartodb_id; +SELECT cartodb_id, the_geom, lat, lon, name FROM test_sync_dest ORDER BY cartodb_id; \echo 'It will work with schemas that need quoting' \set QUIET on diff --git a/test/CDB_SyncTableTest_expect b/test/CDB_SyncTableTest_expect index 01cb358..9e0c8df 100644 --- a/test/CDB_SyncTableTest_expect +++ b/test/CDB_SyncTableTest_expect @@ -54,10 +54,10 @@ NOTICE: MODIFIED 0 row(s) 2|||2|2|bar 4|||4|4|cantaloupe 5|||5|5|sandia -1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo -2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar -4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe -5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia +1|0101000020E6100000000000000000F03F000000000000F03F|1|1|foo +2|0101000020E610000000000000000000400000000000000040|2|2|bar +4|0101000020E610000000000000000010400000000000001040|4|4|cantaloupe +5|0101000020E610000000000000000014400000000000001440|5|5|sandia It will work with schemas that need quoting INSERT 0 1 From 4100b66f3bddf4ccadf2525296abb3c3c92fc3d1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 15:25:57 +0100 Subject: [PATCH 049/108] Scaffolding for diagnostic function(s) --- scripts-available/CDB_FederatedServerDiagnostics.sql | 12 ++++++++++++ .../403-CDB_FederatedServerDiagnostics.sql | 1 + test/CDB_FederatedServerDiagnostics.sql | 2 ++ test/CDB_FederatedServerDiagnostics_expect | 2 ++ 4 files changed, 17 insertions(+) create mode 100644 scripts-available/CDB_FederatedServerDiagnostics.sql create mode 120000 scripts-enabled/403-CDB_FederatedServerDiagnostics.sql create mode 100644 test/CDB_FederatedServerDiagnostics.sql create mode 100644 test/CDB_FederatedServerDiagnostics_expect diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql new file mode 100644 index 0000000..7b16263 --- /dev/null +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -0,0 +1,12 @@ +-------------------------------------------------------------------------------- +-- Public functions +-------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT) +RETURNS json -- TODO decide if json or jsonb +AS $$ +BEGIN + RETURN '{}'::json; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/scripts-enabled/403-CDB_FederatedServerDiagnostics.sql b/scripts-enabled/403-CDB_FederatedServerDiagnostics.sql new file mode 120000 index 0000000..edab4d4 --- /dev/null +++ b/scripts-enabled/403-CDB_FederatedServerDiagnostics.sql @@ -0,0 +1 @@ +../scripts-available/CDB_FederatedServerDiagnostics.sql \ No newline at end of file diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql new file mode 100644 index 0000000..d0af580 --- /dev/null +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -0,0 +1,2 @@ +\echo '%% It raises an error if the server does not exist' +SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist'); diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect new file mode 100644 index 0000000..ea8ac49 --- /dev/null +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -0,0 +1,2 @@ +%% It raises an error if the server does not exist +1.1|{} From 35b2b7e5894a557eeacd77d26dbc59ea48dd20bf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 17:14:43 +0100 Subject: [PATCH 050/108] Check the server is of type PG and return jsonb --- .../CDB_FederatedServerDiagnostics.sql | 30 +++++++++++- test/CDB_FederatedServerDiagnostics.sql | 48 +++++++++++++++++++ test/CDB_FederatedServerDiagnostics_expect | 6 ++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 7b16263..2f727ea 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -1,12 +1,38 @@ +-------------------------------------------------------------------------------- +-- Private functions +-------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name) +RETURNS jsonb +AS $$ +BEGIN + RETURN '{}'::jsonb; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + + -------------------------------------------------------------------------------- -- Public functions -------------------------------------------------------------------------------- +-- +-- TODO: function documentation +-- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT) -RETURNS json -- TODO decide if json or jsonb +RETURNS jsonb AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true); + server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN - RETURN '{}'::json; + CASE server_type + WHEN 'postgres_fdw' THEN + RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal); + ELSE + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; + END CASE; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index d0af580..0c1097e 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -1,2 +1,50 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +CREATE EXTENSION postgres_fdw; + +CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd'; +GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester; + +-- Create database to be used as remote +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); +\set QUIET off + + +-- =================================================================== +-- Test server diagnostics function(s) +-- =================================================================== \echo '%% It raises an error if the server does not exist' SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist'); + +\echo '%% It returns a jsonb object' +SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')); + + +-- =================================================================== +-- Cleanup +-- =================================================================== +\set QUIET on +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +DROP DATABASE cdb_fs_tester; + +-- Drop role +REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester; +DROP ROLE cdb_fs_tester; + +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index ea8ac49..b8ffd9e 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -1,2 +1,6 @@ +C1| %% It raises an error if the server does not exist -1.1|{} +ERROR: Server "doesNotExist" does not exist +%% It returns a jsonb object +1.2|jsonb +D1| From 40a2ba9569df87d9b4d6838e7eba6e2179c9867a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 18:09:22 +0100 Subject: [PATCH 051/108] Stub function to add server_version to diagnostics --- .../CDB_FederatedServerDiagnostics.sql | 15 ++++++++++++++- test/CDB_FederatedServerDiagnostics.sql | 3 +++ test/CDB_FederatedServerDiagnostics_expect | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 2f727ea..fd00e95 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -2,11 +2,24 @@ -- Private functions -------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Version_PG(server_internal name) +RETURNS text +AS $$ +BEGIN + -- TODO Implement + RETURN '14.0'; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name) RETURNS jsonb AS $$ +DECLARE + remote_server_version text := @extschema@.__CDB_FS_Server_Version_PG(server_internal); BEGIN - RETURN '{}'::jsonb; + RETURN jsonb_build_object('server_version', remote_server_version); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 0c1097e..6783118 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -34,6 +34,9 @@ SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist') \echo '%% It returns a jsonb object' SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')); +\echo '%% It returns the server version' +SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_version": "14.0"}'::jsonb; + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index b8ffd9e..9ec5b49 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -3,4 +3,6 @@ C1| ERROR: Server "doesNotExist" does not exist %% It returns a jsonb object 1.2|jsonb +%% It returns the server version +1.3|t D1| From efd757ffa32f2148125893836fac92f48de69455 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 18:18:28 +0100 Subject: [PATCH 052/108] Check against correct server version --- test/CDB_FederatedServerDiagnostics.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 6783118..e26b615 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -35,7 +35,8 @@ SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist') SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')); \echo '%% It returns the server version' -SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_version": "14.0"}'::jsonb; +SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb + FROM pg_settings WHERE name = 'server_version'; -- =================================================================== From 87302920975ff64112ef6964d71cf68b7eaf9cd2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 19:20:08 +0100 Subject: [PATCH 053/108] Function to retrieve remote PG server ver --- .../CDB_FederatedServerDiagnostics.sql | 41 ++++++++++++++++--- test/CDB_FederatedServerDiagnostics.sql | 2 + 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index fd00e95..e55f90f 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -2,22 +2,53 @@ -- Private functions -------------------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Version_PG(server_internal name) +-- +-- Get the version of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name) RETURNS text AS $$ +DECLARE + -- Import pg_settings from pg_catalog + remote_schema name := 'pg_catalog'; + remote_table name := 'pg_settings'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + remote_server_version text; BEGIN - -- TODO Implement - RETURN '14.0'; + -- Import the foreign pg_settings table + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + remote_schema, remote_table, server_internal, local_schema); + END IF; + + BEGIN + EXECUTE format(' + SELECT setting FROM %I.%I WHERE name = ''server_version''; + ', local_schema, remote_table) INTO remote_server_version; + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + END; + + RETURN remote_server_version; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- Collect and return diagnostics info from a remote PG into a jsonb +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name) RETURNS jsonb AS $$ DECLARE - remote_server_version text := @extschema@.__CDB_FS_Server_Version_PG(server_internal); + remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); BEGIN RETURN jsonb_build_object('server_version', remote_server_version); END @@ -31,7 +62,7 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -------------------------------------------------------------------------------- -- --- TODO: function documentation +-- Collect and return diagnostics info from a remote PG into a jsonb -- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT) RETURNS jsonb diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index e26b615..a94776d 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -44,6 +44,8 @@ SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> -- =================================================================== \set QUIET on SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +-- Reconnect, using a new session in order to close FDW connections +\connect DROP DATABASE cdb_fs_tester; -- Drop role From d4bc69cd3c773e0ef6435b66b59ef837f7fe642c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 19:49:16 +0100 Subject: [PATCH 054/108] Return postgis version in diagnostics info (WIP) --- .../CDB_FederatedServerDiagnostics.sql | 19 +++++++++++++++++-- test/CDB_FederatedServerDiagnostics.sql | 4 ++++ test/CDB_FederatedServerDiagnostics_expect | 2 ++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index e55f90f..edc0851 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -41,6 +41,17 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name) +RETURNS text +AS $$ +BEGIN + -- TODO implement + RETURN '4.0'; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + -- -- Collect and return diagnostics info from a remote PG into a jsonb -- @@ -48,9 +59,13 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_int RETURNS jsonb AS $$ DECLARE - remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); + remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); + remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); BEGIN - RETURN jsonb_build_object('server_version', remote_server_version); + RETURN jsonb_build_object( + 'server_version', remote_server_version, + 'postgis_version', remote_postgis_version + ); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index a94776d..864f012 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -38,6 +38,10 @@ SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loop SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb FROM pg_settings WHERE name = 'server_version'; +\echo '%% It returns the postgis version' +SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb + FROM pg_extension WHERE extname = 'postgis'; + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index 9ec5b49..1c58eda 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -5,4 +5,6 @@ ERROR: Server "doesNotExist" does not exist 1.2|jsonb %% It returns the server version 1.3|t +%% It returns the postgis version +1.4|t D1| From 80f01d4a51a2c8593f388e10f14eb784dc3743c4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 20:02:51 +0100 Subject: [PATCH 055/108] Implement retrieval of remote postgis version --- .../CDB_FederatedServerDiagnostics.sql | 32 +++++++++++++++++-- test/CDB_FederatedServerDiagnostics.sql | 4 +++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index edc0851..5b327b2 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -41,12 +41,40 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- Get the PostGIS extension version of a remote PG server +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name) RETURNS text AS $$ +DECLARE + -- Import pg_settings from pg_catalog + remote_schema name := 'pg_catalog'; + remote_table name := 'pg_extension'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + remote_postgis_version text; BEGIN - -- TODO implement - RETURN '4.0'; + -- Import the foreign pg_extension table + IF NOT EXISTS ( + SELECT * FROM pg_class + WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) + AND relname = remote_table + ) THEN + EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', + remote_schema, remote_table, server_internal, local_schema); + END IF; + + BEGIN + EXECUTE format(' + SELECT extversion FROM %I.%I WHERE extname = ''postgis''; + ', local_schema, remote_table) INTO remote_postgis_version; + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Not enough permissions to access the server "%"', + @extschema@.__CDB_FS_Extract_Server_Name(server_internal); + END; + + RETURN remote_postgis_version; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 864f012..e3b880c 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -22,6 +22,10 @@ SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text "password": "cdb_fs_passwd" } }'::jsonb); + +\c cdb_fs_tester postgres +CREATE EXTENSION postgis; +\c contrib_regression postgres \set QUIET off From 511e24a40ebfd60f7bca5cfd5bdbe9e5353983b7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 20:05:08 +0100 Subject: [PATCH 056/108] Remove unused var role_name --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 5b327b2..ee5992e 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -13,7 +13,6 @@ DECLARE remote_schema name := 'pg_catalog'; remote_table name := 'pg_settings'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); remote_server_version text; BEGIN -- Import the foreign pg_settings table @@ -52,7 +51,6 @@ DECLARE remote_schema name := 'pg_catalog'; remote_table name := 'pg_extension'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); remote_postgis_version text; BEGIN -- Import the foreign pg_extension table From 6700d24232ab9bc58b7c90af763c92b21f791309 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 20:16:26 +0100 Subject: [PATCH 057/108] Refactor: extract __CDB_FS_Import_If_Not_Exists --- .../CDB_FederatedServerDiagnostics.sql | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index ee5992e..8f451af 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -3,19 +3,14 @@ -------------------------------------------------------------------------------- -- --- Get the version of a remote PG server +-- Import a foreign table if it does not exist -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name) -RETURNS text +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name) +RETURNS void AS $$ DECLARE - -- Import pg_settings from pg_catalog - remote_schema name := 'pg_catalog'; - remote_table name := 'pg_settings'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - remote_server_version text; BEGIN - -- Import the foreign pg_settings table IF NOT EXISTS ( SELECT * FROM pg_class WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) @@ -24,6 +19,23 @@ BEGIN EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', remote_schema, remote_table, server_internal, local_schema); END IF; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Get the version of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name) +RETURNS text +AS $$ +DECLARE + remote_schema name := 'pg_catalog'; + remote_table name := 'pg_settings'; + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + remote_server_version text; +BEGIN + PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); BEGIN EXECUTE format(' @@ -47,21 +59,12 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(serve RETURNS text AS $$ DECLARE - -- Import pg_settings from pg_catalog remote_schema name := 'pg_catalog'; remote_table name := 'pg_extension'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); remote_postgis_version text; BEGIN - -- Import the foreign pg_extension table - IF NOT EXISTS ( - SELECT * FROM pg_class - WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema) - AND relname = remote_table - ) THEN - EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I', - remote_schema, remote_table, server_internal, local_schema); - END IF; + PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table); BEGIN EXECUTE format(' From eba6cf45659ccb83f6c6aac26f26b184445ddf5e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 8 Nov 2019 20:27:24 +0100 Subject: [PATCH 058/108] Test when postgis is not installed in the remote --- test/CDB_FederatedServerDiagnostics.sql | 8 ++++++++ test/CDB_FederatedServerDiagnostics_expect | 2 ++ 2 files changed, 10 insertions(+) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index e3b880c..ef0dbf1 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -46,6 +46,14 @@ SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb FROM pg_extension WHERE extname = 'postgis'; +\echo '%% It returns null as the postgis version if it is not installed' +\set QUIET on +\c cdb_fs_tester postgres +DROP EXTENSION postgis; +\c contrib_regression postgres +\set QUIET off +SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb; + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index 1c58eda..bff416f 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -7,4 +7,6 @@ ERROR: Server "doesNotExist" does not exist 1.3|t %% It returns the postgis version 1.4|t +%% It returns null as the postgis version if it is not installed +1.5|t D1| From d76e3ccc3e30358bd8bf2442ad0e4e917c58f248 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 15:56:17 +0100 Subject: [PATCH 059/108] Add server options to diagnostics --- .../CDB_FederatedServerDiagnostics.sql | 20 ++++++++++++++++++- test/CDB_FederatedServerDiagnostics.sql | 3 +++ test/CDB_FederatedServerDiagnostics_expect | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 8f451af..e05732d 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -81,6 +81,22 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +-- +-- Get the foreign server options of a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name) +RETURNS jsonb +AS $$ + -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html + SELECT jsonb_object(server_options) FROM ( + SELECT ARRAY(SELECT regexp_split_to_table(opts, '=') + FROM pg_foreign_server, unnest(srvoptions) opts + WHERE srvname = cartodb.__CDB_FS_Generate_Server_Name(input_name => 'loopback', check_existence => true)) server_options + ) q; +$$ +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + + -- -- Collect and return diagnostics info from a remote PG into a jsonb -- @@ -90,10 +106,12 @@ AS $$ DECLARE remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); + remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); BEGIN RETURN jsonb_build_object( 'server_version', remote_server_version, - 'postgis_version', remote_postgis_version + 'postgis_version', remote_postgis_version, + 'server_options', remote_server_options ); END $$ diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index ef0dbf1..f66e563 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -54,6 +54,9 @@ DROP EXTENSION postgis; \set QUIET off SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb; +\echo '%% It returns the remote server options' +SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "5432", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; + -- =================================================================== -- Cleanup diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index bff416f..2f1f7ae 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -9,4 +9,6 @@ ERROR: Server "doesNotExist" does not exist 1.4|t %% It returns null as the postgis version if it is not installed 1.5|t +%% It returns the remote server options +1.6|t D1| From 5c9f6964a321558c6005ab26418f6467d8539251 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 18:46:07 +0100 Subject: [PATCH 060/108] Fix typo: hardcoded loopback server --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index e05732d..abbb0d8 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -91,7 +91,7 @@ AS $$ SELECT jsonb_object(server_options) FROM ( SELECT ARRAY(SELECT regexp_split_to_table(opts, '=') FROM pg_foreign_server, unnest(srvoptions) opts - WHERE srvname = cartodb.__CDB_FS_Generate_Server_Name(input_name => 'loopback', check_existence => true)) server_options + WHERE srvname = server_internal) server_options ) q; $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; From d87d27d7e50685a571467a2438d16fa08163c2f9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 20:42:00 +0100 Subject: [PATCH 061/108] Use pg_options_to_table instead of manual parsing --- .../CDB_FederatedServerDiagnostics.sql | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index abbb0d8..7bf5bee 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -87,14 +87,23 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name) RETURNS jsonb AS $$ +DECLARE + res jsonb := '{}'; + opt record; +BEGIN -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html - SELECT jsonb_object(server_options) FROM ( - SELECT ARRAY(SELECT regexp_split_to_table(opts, '=') - FROM pg_foreign_server, unnest(srvoptions) opts - WHERE srvname = server_internal) server_options - ) q; + -- See https://www.postgresql.org/docs/current/functions-info.html + + -- SELECT jsonb_build_object(opt.option_name, opt.option_value) FROM (SELECT (pg_options_to_table(srvoptions)).* FROM pg_foreign_server WHERE srvname = 'cdb_fdw_local_pg11') opt; + FOR opt IN + SELECT (pg_options_to_table(srvoptions)).* FROM pg_foreign_server WHERE srvname = server_internal + LOOP + res := res || jsonb_build_object(opt.option_name, opt.option_value); + END LOOP; + RETURN res; +END $$ -LANGUAGE SQL VOLATILE PARALLEL UNSAFE; +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- From 6e9ee296c594239db5689c25fbd5553bfcb96a3f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 20:48:14 +0100 Subject: [PATCH 062/108] Remove useless commented code --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 7bf5bee..bf2524e 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -93,8 +93,6 @@ DECLARE BEGIN -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html -- See https://www.postgresql.org/docs/current/functions-info.html - - -- SELECT jsonb_build_object(opt.option_name, opt.option_value) FROM (SELECT (pg_options_to_table(srvoptions)).* FROM pg_foreign_server WHERE srvname = 'cdb_fdw_local_pg11') opt; FOR opt IN SELECT (pg_options_to_table(srvoptions)).* FROM pg_foreign_server WHERE srvname = server_internal LOOP From 0ad7cd485df0c88c5811e04a7e1b2c3545f781dc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 21:34:51 +0100 Subject: [PATCH 063/108] Simplify function with aggregate --- .../CDB_FederatedServerDiagnostics.sql | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index bf2524e..459338d 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -87,21 +87,15 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name) RETURNS jsonb AS $$ -DECLARE - res jsonb := '{}'; - opt record; -BEGIN -- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html -- See https://www.postgresql.org/docs/current/functions-info.html - FOR opt IN - SELECT (pg_options_to_table(srvoptions)).* FROM pg_foreign_server WHERE srvname = server_internal - LOOP - res := res || jsonb_build_object(opt.option_name, opt.option_value); - END LOOP; - RETURN res; -END + SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server + WHERE srvname = server_internal + ) AS opt; $$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- From d5ecb3925041dd3aaf36290ad9e0a32a8a316d38 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 10:32:25 +0100 Subject: [PATCH 064/108] Replace hardcoded 5432 by @@PGPORT@@ as requested in review comment --- test/CDB_FederatedServerDiagnostics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index f66e563..d626004 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -55,7 +55,7 @@ DROP EXTENSION postgis; SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb; \echo '%% It returns the remote server options' -SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "5432", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; +SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; -- =================================================================== From 1c27443ff4e60fc5fd323ed6c63dae3bd2370ccb Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 16:37:27 +0100 Subject: [PATCH 065/108] Stub remote server latency (WIP) --- .../CDB_FederatedServerDiagnostics.sql | 16 +++++++++++++++- test/CDB_FederatedServerDiagnostics.sql | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 459338d..49388c5 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -97,6 +97,18 @@ AS $$ $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; +-- +-- Get the network latency to a remote PG server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal name) +RETURNS float +AS $$ +BEGIN + RETURN 0.0; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + -- -- Collect and return diagnostics info from a remote PG into a jsonb @@ -108,11 +120,13 @@ DECLARE remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); + remote_server_latency float := @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal); BEGIN RETURN jsonb_build_object( 'server_version', remote_server_version, 'postgis_version', remote_postgis_version, - 'server_options', remote_server_options + 'server_options', remote_server_options, + 'server_latency', remote_server_latency ); END $$ diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index d626004..e48fcb3 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -57,6 +57,9 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> \echo '%% It returns the remote server options' SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; +\echo '%% It returns the network latency to the remote server' +SELECT '1.7', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_latency": 0.0}'::jsonb; + -- =================================================================== -- Cleanup From dbed2d1edeee1e87478d1b4b1887c078fcae9a9a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 11 Nov 2019 18:03:14 +0100 Subject: [PATCH 066/108] More scaffolding for latency in diagnostics --- .../CDB_FederatedServerDiagnostics.sql | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 49388c5..2ba9092 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -97,14 +97,57 @@ AS $$ $$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; + +-- +-- Get a foreign PG server hostname from the catalog +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name) +RETURNS text +AS $$ +BEGIN + -- TODO: implement + RETURN 'localhost'; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +-- +-- Get a foreign PG server port from the catalog +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name) +RETURNS integer +AS $$ +BEGIN + -- TODO: implement + RETURN 5432; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + +-- +-- Get one measure of network latency to a remote TCP server +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Network_Latency(host text, port integer) +RETURNS float +AS $$ + #--TODO implement + return 0.0 +$$ +LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + -- -- Get the network latency to a remote PG server -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal name) RETURNS float AS $$ +DECLARE + remote_server_host text := @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal); + remote_server_port integer := @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal); -- TODO: best type for ports + latency float; BEGIN - RETURN 0.0; + latency := @extschema@.__CDB_FS_TCP_Network_Latency(host => remote_server_host, port => remote_server_port); + RETURN latency; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; From 8c7a75dd0ccd1a07ccc00a19dd7ab5c862213826 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 12:40:53 +0100 Subject: [PATCH 067/108] Implement __CDB_FS_TCP_Network_Latency --- .../CDB_FederatedServerDiagnostics.sql | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 2ba9092..ca36001 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -126,12 +126,31 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- --- Get one measure of network latency to a remote TCP server -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Network_Latency(host text, port integer) +-- Get one measure of network latency in ms to a remote TCP server +-- +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Network_Latency( + host text, + port integer DEFAULT 5432, + timeout_seconds float DEFAULT 5.0 +) RETURNS float AS $$ - #--TODO implement - return 0.0 + import socket + from timeit import default_timer as timer + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout_seconds) + + t_start = timer() + + try: + s.connect((host, int(port))) + t_stop = timer() + s.shutdown(socket.SHUT_RD) + except (socket.timeout, OSError, socket.error): + return None + + return (t_stop - t_start) * 1000.0 $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; @@ -143,7 +162,7 @@ RETURNS float AS $$ DECLARE remote_server_host text := @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal); - remote_server_port integer := @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal); -- TODO: best type for ports + remote_server_port integer := @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal); latency float; BEGIN latency := @extschema@.__CDB_FS_TCP_Network_Latency(host => remote_server_host, port => remote_server_port); From e14c49b22357ca938310d0eea69e7f4df54a899d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 12:59:36 +0100 Subject: [PATCH 068/108] Add kind of meaningful test for latency --- test/CDB_FederatedServerDiagnostics.sql | 3 ++- test/CDB_FederatedServerDiagnostics_expect | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index e48fcb3..0043769 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -58,7 +58,8 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; \echo '%% It returns the network latency to the remote server' -SELECT '1.7', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_latency": 0.0}'::jsonb; +SELECT '1.7', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float > 0.0; +SELECT '1.8', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float < 1000.0; -- =================================================================== diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index 2f1f7ae..d2c543e 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -11,4 +11,7 @@ ERROR: Server "doesNotExist" does not exist 1.5|t %% It returns the remote server options 1.6|t +%% It returns the network latency to the remote server +1.7|t +1.8|t D1| From 43b242e61021bb1af1f5f1ecbca3054ce730106f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 13:02:04 +0100 Subject: [PATCH 069/108] Retrieve host from the catalog --- scripts-available/CDB_FederatedServerDiagnostics.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index ca36001..b794440 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -104,12 +104,12 @@ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name) RETURNS text AS $$ -BEGIN - -- TODO: implement - RETURN 'localhost'; -END + SELECT option_value FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server WHERE srvname = server_internal + ) AS opt WHERE opt.option_name = 'host'; $$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- From 26f91001ee117f08c034e5b305d4342075e10a15 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 13:06:14 +0100 Subject: [PATCH 070/108] Retrieve port from the catalog --- scripts-available/CDB_FederatedServerDiagnostics.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index b794440..67398e3 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -118,12 +118,12 @@ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name) RETURNS integer AS $$ -BEGIN - -- TODO: implement - RETURN 5432; -END + SELECT option_value::integer FROM ( + SELECT (pg_options_to_table(srvoptions)).* + FROM pg_foreign_server WHERE srvname = server_internal + ) AS opt WHERE opt.option_name = 'port'; $$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; +LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- -- Get one measure of network latency in ms to a remote TCP server From d3ae4b808ced0340f021b2ba6e0cc5bb296dc70d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 13:27:42 +0100 Subject: [PATCH 071/108] Remove default port 5432 in generic function --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 67398e3..21d3147 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -130,7 +130,7 @@ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Network_Latency( host text, - port integer DEFAULT 5432, + port integer, timeout_seconds float DEFAULT 5.0 ) RETURNS float From 9dc547c83855f8e82bc8d88c49bacf6cc55a8663 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 13:37:04 +0100 Subject: [PATCH 072/108] Add test for wrong port --- test/CDB_FederatedServerDiagnostics.sql | 19 +++++++++++++++++-- test/CDB_FederatedServerDiagnostics_expect | 8 ++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 0043769..47c27bc 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -58,8 +58,22 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; \echo '%% It returns the network latency to the remote server' -SELECT '1.7', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float > 0.0; -SELECT '1.8', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float < 1000.0; +SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float > 0.0; +SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float < 1000.0; + +\echo '%% It raises an error if the wrong port is provided' +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ + "server": { + "host": "localhost", + "port": "12345" + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); +SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port'); + -- =================================================================== @@ -67,6 +81,7 @@ SELECT '1.8', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->' -- =================================================================== \set QUIET on SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); +SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text); -- Reconnect, using a new session in order to close FDW connections \connect DROP DATABASE cdb_fs_tester; diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index d2c543e..e030def 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -12,6 +12,10 @@ ERROR: Server "doesNotExist" does not exist %% It returns the remote server options 1.6|t %% It returns the network latency to the remote server -1.7|t -1.8|t +2.1|t +2.2|t +%% It raises an error if the wrong port is provided +C2| +ERROR: could not connect to server "cdb_fs_wrong-port" D1| +D2| From a032cc08e08ad961e73a23fc990a0e1cbde9000b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 15:01:56 +0100 Subject: [PATCH 073/108] Measure latency as the avg of n_samples --- scripts-available/CDB_FederatedServerDiagnostics.sql | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 21d3147..5700b0c 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -148,16 +148,19 @@ AS $$ t_stop = timer() s.shutdown(socket.SHUT_RD) except (socket.timeout, OSError, socket.error): + plpy.warning('could not connect to server: %s:%d' % (host, port)) return None - return (t_stop - t_start) * 1000.0 + t_total = (t_stop - t_start) * 1000.0 + plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_total)) + return t_total $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; -- -- Get the network latency to a remote PG server -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal name) +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal name, n_samples integer DEFAULT 10) RETURNS float AS $$ DECLARE @@ -165,7 +168,10 @@ DECLARE remote_server_port integer := @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal); latency float; BEGIN - latency := @extschema@.__CDB_FS_TCP_Network_Latency(host => remote_server_host, port => remote_server_port); + latency := avg(sample) FROM ( + SELECT @extschema@.__CDB_FS_TCP_Network_Latency(host => remote_server_host, port => remote_server_port) sample + FROM generate_series(1, n_samples) + ) q; RETURN latency; END $$ From 18f9f79fe8ae2568fd28309711a1a315c44f0cd4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 15:08:26 +0100 Subject: [PATCH 074/108] Rename key to server_latency_ms to be self-explanatory --- scripts-available/CDB_FederatedServerDiagnostics.sql | 4 ++-- test/CDB_FederatedServerDiagnostics.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 5700b0c..3a4edf3 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -188,13 +188,13 @@ DECLARE remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); - remote_server_latency float := @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal); + remote_server_latency_ms float := @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal); BEGIN RETURN jsonb_build_object( 'server_version', remote_server_version, 'postgis_version', remote_postgis_version, 'server_options', remote_server_options, - 'server_latency', remote_server_latency + 'server_latency_ms', remote_server_latency_ms ); END $$ diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 47c27bc..1d803d5 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -58,8 +58,8 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; \echo '%% It returns the network latency to the remote server' -SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float > 0.0; -SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency')::float < 1000.0; +SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::float > 0.0; +SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::float < 1000.0; \echo '%% It raises an error if the wrong port is provided' SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ From 364933b6b983064a1efab6ad6d9c7c735c241583 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 12 Nov 2019 15:57:15 +0100 Subject: [PATCH 075/108] Compatibility with PG 9.6 and 10 See https://www.postgresql.org/message-id/flat/55137679-fd6b-baf0-bfbf-6558ce5cb409%40postgrespro.ru --- test/CDB_FederatedServerDiagnostics.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 1d803d5..9e40773 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -58,8 +58,8 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; \echo '%% It returns the network latency to the remote server' -SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::float > 0.0; -SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::float < 1000.0; +SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::text::float > 0.0; +SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::text::float < 1000.0; \echo '%% It raises an error if the wrong port is provided' SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ From 0d455d9521411e810e13dd5b7e8f92f4760443be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 12 Nov 2019 18:14:36 +0100 Subject: [PATCH 076/108] __CDB_FS_List_Foreign_Schemas_PG: Remove extra role generation --- scripts-available/CDB_FederatedServerListRemote.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index a7085ed..0367027 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -22,7 +22,6 @@ DECLARE inf_schema name := 'information_schema'; remote_table name := 'schemata'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); - role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN -- Import the foreign schemata table IF NOT EXISTS ( From 9eab5e25337711cddb813e225d86dcc7bba96b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 12 Nov 2019 18:15:20 +0100 Subject: [PATCH 077/108] __CDB_FS_Generate_Server_Role_Name: set to STABLE current_database() is stable, not immutable --- scripts-available/CDB_FederatedServer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 32ec32a..6d3549a 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -87,7 +87,7 @@ BEGIN RETURN role_name::name; END $$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +LANGUAGE PLPGSQL STABLE PARALLEL SAFE; -- -- Creates (if not exist) a schema to place the objects for a remote schema From 0362825b8c0ccfed303cdd8f74d4a6238f142f4d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 11:44:24 +0100 Subject: [PATCH 078/108] Encapsulate python in __CDB_FS_TCP_Foreign_Server_Latency Instead of having a generic function, merge __CDB_FS_TCP_Network_Latency and __CDB_FS_Foreign_Server_Host_PG into one function so that it can only "ping" federated servers. --- .../CDB_FederatedServerDiagnostics.sql | 44 +++++++------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 3a4edf3..8a9188a 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -128,16 +128,24 @@ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- -- Get one measure of network latency in ms to a remote TCP server -- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Network_Latency( - host text, - port integer, - timeout_seconds float DEFAULT 5.0 +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency( + server_internal name, + timeout_seconds integer DEFAULT 5.0, + n_samples integer DEFAULT 10 ) RETURNS float AS $$ import socket from timeit import default_timer as timer + plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name']) + rv = plpy.execute(plan, [server_internal], 1) + host = rv[0]['host'] + + plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name']) + rv = plpy.execute(plan, [server_internal], 1) + port = rv[0]['port'] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout_seconds) @@ -157,26 +165,6 @@ AS $$ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; --- --- Get the network latency to a remote PG server --- -CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal name, n_samples integer DEFAULT 10) -RETURNS float -AS $$ -DECLARE - remote_server_host text := @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal); - remote_server_port integer := @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal); - latency float; -BEGIN - latency := avg(sample) FROM ( - SELECT @extschema@.__CDB_FS_TCP_Network_Latency(host => remote_server_host, port => remote_server_port) sample - FROM generate_series(1, n_samples) - ) q; - RETURN latency; -END -$$ -LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; - -- -- Collect and return diagnostics info from a remote PG into a jsonb @@ -185,10 +173,10 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_int RETURNS jsonb AS $$ DECLARE - remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); - remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); - remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); - remote_server_latency_ms float := @extschema@.__CDB_FS_Foreign_Server_Latency_PG(server_internal); + remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); + remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); + remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); + remote_server_latency_ms float := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal); BEGIN RETURN jsonb_build_object( 'server_version', remote_server_version, From a4a4a1dff13287316b7d6f256065662b8957aa7e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 12:13:42 +0100 Subject: [PATCH 079/108] Honor n_samples param --- .../CDB_FederatedServerDiagnostics.sql | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 8a9188a..f0205f8 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -130,7 +130,7 @@ LANGUAGE SQL VOLATILE PARALLEL UNSAFE; -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency( server_internal name, - timeout_seconds integer DEFAULT 5.0, + timeout_seconds float DEFAULT 5.0, n_samples integer DEFAULT 10 ) RETURNS float @@ -150,18 +150,22 @@ AS $$ s.settimeout(timeout_seconds) t_start = timer() + samples = [] - try: - s.connect((host, int(port))) - t_stop = timer() - s.shutdown(socket.SHUT_RD) - except (socket.timeout, OSError, socket.error): - plpy.warning('could not connect to server: %s:%d' % (host, port)) - return None + for i in xrange(n_samples): + try: + s.connect((host, int(port))) + t_stop = timer() + s.shutdown(socket.SHUT_RD) + except (socket.timeout, OSError, socket.error), ex: + plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) + samples.append(None) - t_total = (t_stop - t_start) * 1000.0 - plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_total)) - return t_total + t_connect = (t_stop - t_start) * 1000.0 + plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) + samples.append(t_connect) + + return sum(samples) / n_samples $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; From 980657c4a425a17a22d58b3957f215db23958afc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 12:16:21 +0100 Subject: [PATCH 080/108] Do not add None to samples in case of connection error Otherwise we get `TypeError: unsupported operand type(s) for +: 'float' and 'NoneType'` when calculating the average. --- scripts-available/CDB_FederatedServerDiagnostics.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index f0205f8..4904a38 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -151,6 +151,7 @@ AS $$ t_start = timer() samples = [] + n_errors = 0 for i in xrange(n_samples): try: @@ -159,13 +160,13 @@ AS $$ s.shutdown(socket.SHUT_RD) except (socket.timeout, OSError, socket.error), ex: plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) - samples.append(None) + n_errors += 1 t_connect = (t_stop - t_start) * 1000.0 plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) samples.append(t_connect) - return sum(samples) / n_samples + return sum(samples) / len(samples) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; From 5897d8a9cb6b3edbabcf405fa1cebbe866b3643d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 12:24:59 +0100 Subject: [PATCH 081/108] Fix timer start in measurements --- .../CDB_FederatedServerDiagnostics.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 4904a38..f33d22b 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -146,14 +146,14 @@ AS $$ rv = plpy.execute(plan, [server_internal], 1) port = rv[0]['port'] - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(timeout_seconds) - - t_start = timer() - samples = [] - n_errors = 0 - for i in xrange(n_samples): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout_seconds) + + t_start = timer() + samples = [] + n_errors = 0 + try: s.connect((host, int(port))) t_stop = timer() From 4b9c048235ff5ea2d95cae26f008a9eb4c28af26 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 12:25:41 +0100 Subject: [PATCH 082/108] Close socket properly before the next iteration This avoid the errors `[Errno 106] Transport endpoint is already connected` which are due to connections kept in TIME_WAIT state. We don't use socket.SO_REUSEADDR as it would make the first connection a special case compared to the following ones. --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index f33d22b..05aac38 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -161,6 +161,8 @@ AS $$ except (socket.timeout, OSError, socket.error), ex: plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) n_errors += 1 + finally: + s.close() t_connect = (t_stop - t_start) * 1000.0 plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) From e231500c469eb7f853129e55f91361bf7a127238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 13 Nov 2019 14:08:31 +0100 Subject: [PATCH 083/108] Give permissions over cartodb to the FS role --- scripts-available/CDB_FederatedServer.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 6d3549a..c496e59 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -238,6 +238,8 @@ BEGIN EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); END IF; EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); + EXECUTE FORMAT('GRANT ALL ON SCHEMA %I TO %I', '@extschema@', role_name); + EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS in SCHEMA %I TO %I', '@extschema@', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); From 19c6de2096fbb63395055d8f1b331eba68f70944 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 14:52:24 +0100 Subject: [PATCH 084/108] Comment out suspected line This may be causing the error ERROR: could not compile PL/Python function "__cdb_fs_tcp_foreign_server_latency" DETAIL: SyntaxError: invalid syntax (, line 26) when testing in PG12 + plpythonu3 --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 05aac38..cfa845e 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -159,7 +159,7 @@ AS $$ t_stop = timer() s.shutdown(socket.SHUT_RD) except (socket.timeout, OSError, socket.error), ex: - plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) + #-- TODO uncomment & fix: plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) n_errors += 1 finally: s.close() From 203bd64058d4af05d40200fdc2c7be856c306a67 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 17:02:22 +0100 Subject: [PATCH 085/108] Fix python2/3 compat issue --- scripts-available/CDB_FederatedServerDiagnostics.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index cfa845e..f2df36c 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -158,8 +158,8 @@ AS $$ s.connect((host, int(port))) t_stop = timer() s.shutdown(socket.SHUT_RD) - except (socket.timeout, OSError, socket.error), ex: - #-- TODO uncomment & fix: plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) + except (socket.timeout, OSError, socket.error) as ex: + plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex))) n_errors += 1 finally: s.close() From f7d10adbc25784aa1259e350449c542fe5913b16 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 17:16:53 +0100 Subject: [PATCH 086/108] Fix another python2/3 incompat: xrange --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index f2df36c..29b06ee 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -146,7 +146,7 @@ AS $$ rv = plpy.execute(plan, [server_internal], 1) port = rv[0]['port'] - for i in xrange(n_samples): + for i in range(n_samples): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout_seconds) From 3ae7b1fe055587aceb93b79ffd13a6a203c02eae Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 18:27:14 +0100 Subject: [PATCH 087/108] Fix bug: do not initialize samples in every iter --- scripts-available/CDB_FederatedServerDiagnostics.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 29b06ee..5bd4ab8 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -146,13 +146,14 @@ AS $$ rv = plpy.execute(plan, [server_internal], 1) port = rv[0]['port'] + n_errors = 0 + samples = [] + for i in range(n_samples): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(timeout_seconds) t_start = timer() - samples = [] - n_errors = 0 try: s.connect((host, int(port))) From 57dcfc2a35610d423d146456591e98e436c80c03 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 18:55:09 +0100 Subject: [PATCH 088/108] Add a bunch of statistics --- .../CDB_FederatedServerDiagnostics.sql | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 5bd4ab8..cce5038 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -133,9 +133,11 @@ CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency( timeout_seconds float DEFAULT 5.0, n_samples integer DEFAULT 10 ) -RETURNS float +RETURNS jsonb AS $$ import socket + import json + import math from timeit import default_timer as timer plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name']) @@ -169,7 +171,21 @@ AS $$ plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) samples.append(t_connect) - return sum(samples) / len(samples) + # --stats + n = len(samples) + mean = sum(samples) / n + var = sum([ (x - mean)**2 for x in samples ]) / (n-1) + stdev = math.sqrt(var) + + ret = { + 'n_samples': n_samples, + 'n_errors': n_errors, + 'avg': round(mean, 3), + 'min': round(min(samples), 3), + 'max': round(max(samples), 3), + 'stdev': round(stdev, 3) + } + return json.dumps(ret) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; @@ -184,7 +200,7 @@ DECLARE remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal); remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal); remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal); - remote_server_latency_ms float := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal); + remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal); BEGIN RETURN jsonb_build_object( 'server_version', remote_server_version, From a340f98b96e0b3c5f19e82962ff4ac581eb0c1a1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 19:19:39 +0100 Subject: [PATCH 089/108] Update tests with latency stats --- test/CDB_FederatedServerDiagnostics.sql | 22 +++++++++++++++++++--- test/CDB_FederatedServerDiagnostics_expect | 9 ++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index 9e40773..aadb124 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -57,9 +57,25 @@ SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> \echo '%% It returns the remote server options' SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb; -\echo '%% It returns the network latency to the remote server' -SELECT '2.1', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::text::float > 0.0; -SELECT '2.2', (cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback')->'server_latency_ms')::text::float < 1000.0; +\echo '%% It returns network latency stats to the remote server: min <= avg <= max' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.1', (latency.ms->'min')::text::float <= (latency.ms->'avg')::text::float, (latency.ms->'avg')::text::float <= (latency.ms->'max')::text::float +FROM latency; + +\echo '%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.2', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 +FROM latency; + +\echo '%% Latency stats: stdev > 0' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms +) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0 +FROM latency; + + \echo '%% It raises an error if the wrong port is provided' SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index e030def..df01e73 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -11,9 +11,12 @@ ERROR: Server "doesNotExist" does not exist 1.5|t %% It returns the remote server options 1.6|t -%% It returns the network latency to the remote server -2.1|t -2.2|t +%% It returns network latency stats to the remote server: min <= avg <= max +2.1|t|t +%% Latency stats: 0 <= min <= max <= 1000 ms (local connection) +2.2|t|t +%% Latency stats: stdev > 0 +2.3|t %% It raises an error if the wrong port is provided C2| ERROR: could not connect to server "cdb_fs_wrong-port" From 9b9170448081c075f6fc3f2ce34c5e1fd60a5474 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 13 Nov 2019 19:27:56 +0100 Subject: [PATCH 090/108] Calculate stats only when they make sense --- .../CDB_FederatedServerDiagnostics.sql | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index cce5038..6c5d369 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -171,21 +171,25 @@ AS $$ plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect)) samples.append(t_connect) - # --stats - n = len(samples) - mean = sum(samples) / n - var = sum([ (x - mean)**2 for x in samples ]) / (n-1) - stdev = math.sqrt(var) - - ret = { + stats = { 'n_samples': n_samples, 'n_errors': n_errors, - 'avg': round(mean, 3), - 'min': round(min(samples), 3), - 'max': round(max(samples), 3), - 'stdev': round(stdev, 3) } - return json.dumps(ret) + n = len(samples) + if n >= 1: + mean = sum(samples) / n + stats.update({ + 'avg': round(mean, 3), + 'min': round(min(samples), 3), + 'max': round(max(samples), 3) + }) + if n >= 2: + var = sum([ (x - mean)**2 for x in samples ]) / (n-1) + stdev = math.sqrt(var) + stats.update({ + 'stdev': round(stdev, 3) + }) + return json.dumps(stats) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; From 49200295600be2b4e4c60f1e720b8a7cb1f41d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 13:10:21 +0100 Subject: [PATCH 091/108] CDB_FederatedServerTables: Improve permission handling and error messages --- .../CDB_FederatedServerTables.sql | 21 +++++++++++++------ test/CDB_FederatedServerTables_expect | 6 +++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index c54cf06..8356340 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -227,7 +227,7 @@ BEGIN EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', remote_schema, remote_table, server_internal, local_schema); EXCEPTION WHEN OTHERS THEN - RAISE EXCEPTION 'Could not import schema "%" of server "%"', remote_schema, server; + RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM; END; BEGIN @@ -284,6 +284,10 @@ BEGIN webmercator_expression ]; + -- To create the view we switch to the caller role to make sure we have permissions + -- to write in the destination schema + RESET ROLE; + -- Create a view with homogeneous CDB fields BEGIN EXECUTE format( @@ -294,12 +298,17 @@ BEGIN array_to_string(carto_columns_expression || rest_of_cols, ','), src_table ); - EXCEPTION - WHEN insufficient_privilege THEN - RAISE EXCEPTION 'Could not import table "%" as "%": "%" already exists', remote_table, local_name, local_name; - WHEN OTHERS THEN - RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; + EXCEPTION WHEN OTHERS THEN + IF EXISTS (SELECT to_regclass(local_name)) THEN + RAISE EXCEPTION 'Could not import table "%" as "%" already exists: %', remote_table, local_name, SQLERRM; + ELSE + RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; + END IF; END; + + EXECUTE format('ALTER VIEW %1$I OWNER TO %I', + local_name, + cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal)); END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 8267a72..b8f9abc 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -27,7 +27,7 @@ ERROR: Server "Does not exist" does not exist ## Registering a table: NULL server fails ERROR: Server name cannot be NULL ## Registering a table: Invalid schema fails -ERROR: Could not import schema "Does not exist" of server "loopback" +ERROR: Could not import schema "Does not exist" of server "loopback": schema "Does not exist" is not present on foreign server "cdb_fs_loopback" ## Registering a table: NULL schema fails ERROR: Schema name cannot be NULL ## Registering a table: Invalid table fails @@ -50,10 +50,10 @@ ERROR: non geometry column "Does not exists" ## Target conflict is handled nicely: Table CREATE TABLE -ERROR: Could not import table "remote_geom" as "localtable": "localtable" already exists +ERROR: Could not import table "remote_geom" as "localtable" already exists: "localtable" is not a view ## Target conflict is handled nicely: View CREATE VIEW -ERROR: Could not import table "remote_geom" as "localtable2": "localtable2" already exists +ERROR: Could not import table "remote_geom" as "localtable2" already exists: cannot change name of view column "a" to "cartodb_id" DROP VIEW DROP TABLE ## Registering tables does not work without permissions From 38d8a470bc104c7f2cc403380f5b5b8191463748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 13:21:25 +0100 Subject: [PATCH 092/108] Federated Server: Grant all over postgis and cartodb schemas --- scripts-available/CDB_FederatedServer.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index c496e59..1948481 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -238,8 +238,14 @@ BEGIN EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); END IF; EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); - EXECUTE FORMAT('GRANT ALL ON SCHEMA %I TO %I', '@extschema@', role_name); - EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS in SCHEMA %I TO %I', '@extschema@', role_name); + + EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name); + EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name); + + EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name); + EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name); + EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name); + EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); From 64e185b8415b91a794ff84b45028868181b74572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 13:43:28 +0100 Subject: [PATCH 093/108] Go back to public user mappings --- scripts-available/CDB_FederatedServer.sql | 10 +++++----- scripts-available/CDB_FederatedServerTables.sql | 10 +++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 1948481..6562756 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -250,7 +250,7 @@ BEGIN EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); - EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal); + EXECUTE FORMAT ('CREATE USER MAPPING FOR PUBLIC SERVER %I', server_internal); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM USING HINT = 'Please clean the left over objects'; @@ -276,12 +276,12 @@ BEGIN LOOP IF NOT EXISTS ( WITH a AS ( - SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name + SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = 'public' ) SELECT * from a where options = option.key) THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', server_internal, option.key, option.value); ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', server_internal, option.key, option.value); END IF; END LOOP; END @@ -300,7 +300,7 @@ DECLARE BEGIN SET client_min_messages = ERROR; BEGIN - EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); + EXECUTE FORMAT ('DROP USER MAPPING FOR PUBLIC SERVER %I', server_internal); EXECUTE FORMAT ('DROP OWNED BY %I', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 8356340..468bc1c 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -306,9 +306,13 @@ BEGIN END IF; END; - EXECUTE format('ALTER VIEW %1$I OWNER TO %I', - local_name, - cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal)); + BEGIN + EXECUTE format('ALTER VIEW %1$I OWNER TO %I', + local_name, + cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal)); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'View "%" could not be shared with other users: %', local_name, SQLERRM; + END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; From 1b8651700710984b13f4a61ca0dfe8c9451f9117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 14:07:41 +0100 Subject: [PATCH 094/108] CDB_Federated_Server_Unregister: Drop owned in cascade Since now views might be owned by other roles, we need to drop them in cascade to force deletion --- scripts-available/CDB_FederatedServer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 6562756..29e36ae 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -301,7 +301,7 @@ BEGIN SET client_min_messages = ERROR; BEGIN EXECUTE FORMAT ('DROP USER MAPPING FOR PUBLIC SERVER %I', server_internal); - EXECUTE FORMAT ('DROP OWNED BY %I', role_name); + EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; From 6e455efcddf973ef154d965cc3b54105da5c82f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 14:15:51 +0100 Subject: [PATCH 095/108] Improve warning --- scripts-available/CDB_FederatedServerTables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 468bc1c..553afca 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -311,7 +311,8 @@ BEGIN local_name, cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal)); EXCEPTION WHEN OTHERS THEN - RAISE WARNING 'View "%" could not be shared with other users: %', local_name, SQLERRM; + RAISE WARNING 'View "%" could not be shared with other users: %. Execute "ALTER VIEW %1$I OWNER TO %I" to share it', + local_name, SQLERRM, local_name, cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal); END; END $$ From 7f9905cbd5979a2902a61105f4f2257611d53ad1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 14 Nov 2019 17:56:29 +0100 Subject: [PATCH 096/108] Test what happens when missing PG port --- test/CDB_FederatedServerDiagnostics.sql | 39 +++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/test/CDB_FederatedServerDiagnostics.sql b/test/CDB_FederatedServerDiagnostics.sql index aadb124..da5e8cb 100644 --- a/test/CDB_FederatedServerDiagnostics.sql +++ b/test/CDB_FederatedServerDiagnostics.sql @@ -23,6 +23,27 @@ SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text } }'::jsonb); +SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ + "server": { + "host": "localhost", + "port": "12345" + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + +SELECT 'C3', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback-no-port'::text, config => '{ + "server": { + "host": "localhost" + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + \c cdb_fs_tester postgres CREATE EXTENSION postgis; \c contrib_regression postgres @@ -75,21 +96,14 @@ WITH latency AS ( ) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0 FROM latency; - - \echo '%% It raises an error if the wrong port is provided' -SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{ - "server": { - "host": "localhost", - "port": "12345" - }, - "credentials": { - "username": "cdb_fs_tester", - "password": "cdb_fs_passwd" - } -}'::jsonb); SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port'); +\echo '%% Latency stats: can get them on default PG port 5432 when not provided' +WITH latency AS ( + SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms +) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0 +FROM latency; -- =================================================================== @@ -98,6 +112,7 @@ SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port'); \set QUIET on SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text); SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text); +SELECT 'D3', cartodb.CDB_Federated_Server_Unregister(server => 'loopback-no-port'::text); -- Reconnect, using a new session in order to close FDW connections \connect DROP DATABASE cdb_fs_tester; From 064cc2a76bd515eeab00edb8d38f5fe46fecbb31 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 14 Nov 2019 17:57:05 +0100 Subject: [PATCH 097/108] Fix for missing port in pg_foreign_server: default to standard 5432 --- scripts-available/CDB_FederatedServerDiagnostics.sql | 2 +- test/CDB_FederatedServerDiagnostics_expect | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index 6c5d369..b1e2e4c 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -146,7 +146,7 @@ AS $$ plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name']) rv = plpy.execute(plan, [server_internal], 1) - port = rv[0]['port'] + port = rv[0]['port'] or 5432 n_errors = 0 samples = [] diff --git a/test/CDB_FederatedServerDiagnostics_expect b/test/CDB_FederatedServerDiagnostics_expect index df01e73..044ad2f 100644 --- a/test/CDB_FederatedServerDiagnostics_expect +++ b/test/CDB_FederatedServerDiagnostics_expect @@ -1,4 +1,6 @@ C1| +C2| +C3| %% It raises an error if the server does not exist ERROR: Server "doesNotExist" does not exist %% It returns a jsonb object @@ -18,7 +20,9 @@ ERROR: Server "doesNotExist" does not exist %% Latency stats: stdev > 0 2.3|t %% It raises an error if the wrong port is provided -C2| ERROR: could not connect to server "cdb_fs_wrong-port" +%% Latency stats: can get them on default PG port 5432 when not provided +2.4|t|t D1| D2| +D3| From e69d3b1c30c6e178c79df4c9d6d0691e2aa2897e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 18:45:09 +0100 Subject: [PATCH 098/108] Place all views in a shared schema --- scripts-available/CDB_FederatedServer.sql | 14 ++++++-- .../CDB_FederatedServerTables.sql | 35 +++++++++---------- test/CDB_FederatedServerTables.sql | 26 ++++++++------ test/CDB_FederatedServerTables_expect | 28 ++++++++------- 4 files changed, 57 insertions(+), 46 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 29e36ae..bdf8d3a 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -5,9 +5,16 @@ -- -- This function is just a placement to store and use the pattern for -- foreign object names --- Servers: cdb_fs_$(server_name) --- Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) --- Owner role: cdb_fs_$(md5sum(current_database() || server_name) +-- Servers: cdb_fs_$(server_name) +-- View schema: cdb_fs_$(server_name) +-- > This is where all views created when importing tables are placed +-- > One server has only one view schema +-- Import Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name)) +-- > This is where the foreign tables are placed +-- > One server has one import schema per remote schema plus auxiliar ones used +-- to access the remote catalog (pg_catalog, information_schema...) +-- Owner role: cdb_fs_$(md5sum(current_database() || server_name) +-- > This is the role than owns all schemas and tables related to the server -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern() RETURNS TEXT @@ -19,6 +26,7 @@ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- -- Produce a valid DB name for servers generated for the Federated Server -- If check_existence is true, it'll throw if the server doesn't exists +-- This name is also used to -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) RETURNS NAME diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 553afca..013de5e 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -173,6 +173,8 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- can be used through SQL and Maps API's. -- If the table was already exported, it will be dropped and re-imported -- +-- The view is placed under the server's view schema (cdb_fs_$(server_name)) +-- -- E.g: -- SELECT cartodb.CDB_SetUp_PG_Federated_Table( -- 'amazon', -- mandatory, name of the federated server @@ -198,7 +200,10 @@ AS $$ DECLARE server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false); local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); - src_table REGCLASS; + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); + + src_table REGCLASS; -- import_schema.remote_table - import_schema is local_schema + local_view text; -- view_schema.local_name - view_schema is server_internal rest_of_cols TEXT[]; geom_expression TEXT; @@ -284,36 +289,28 @@ BEGIN webmercator_expression ]; - -- To create the view we switch to the caller role to make sure we have permissions - -- to write in the destination schema - RESET ROLE; + -- Create view schema if it doesn't exist + IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN + EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name); + END IF; -- Create a view with homogeneous CDB fields BEGIN EXECUTE format( - 'CREATE OR REPLACE VIEW %1$I AS - SELECT %2s - FROM %3$s t', - local_name, + 'CREATE OR REPLACE VIEW %1$I.%2$I AS + SELECT %3s + FROM %4$s t', + server_internal, local_name, array_to_string(carto_columns_expression || rest_of_cols, ','), src_table ); EXCEPTION WHEN OTHERS THEN - IF EXISTS (SELECT to_regclass(local_name)) THEN - RAISE EXCEPTION 'Could not import table "%" as "%" already exists: %', remote_table, local_name, SQLERRM; + IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN + RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name; ELSE RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; END IF; END; - - BEGIN - EXECUTE format('ALTER VIEW %1$I OWNER TO %I', - local_name, - cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal)); - EXCEPTION WHEN OTHERS THEN - RAISE WARNING 'View "%" could not be shared with other users: %. Execute "ALTER VIEW %1$I OWNER TO %I" to share it', - local_name, SQLERRM, local_name, cartodb.__CDB_FS_Generate_Server_Role_Name(server_internal); - END; END $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 328a05b..fdd1d7a 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -71,7 +71,8 @@ SELECT 'R1', cartodb.CDB_Federated_Table_Register( geom_column => 'geom' ); -SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; +SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback.remote_geom; + Select * FROM CDB_Federated_Server_List_Remote_Tables( server => 'loopback', remote_schema => 'remote_schema' @@ -88,7 +89,7 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( local_name => 'myFullTable' ); -SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM "myFullTable"; +SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback."myFullTable"; Select * FROM CDB_Federated_Server_List_Remote_Tables( server => 'loopback', remote_schema => 'remote_schema' @@ -107,9 +108,9 @@ SELECT 'R3', cartodb.CDB_Federated_Table_Register( ); -- The old view should dissapear -SELECT 'S3_old', cartodb_id, another_field FROM "myFullTable"; +SELECT 'S3_old', cartodb_id, another_field FROM cdb_fs_loopback."myFullTable"; -- And the new appear -SELECT 'S3_new', cartodb_id, another_field FROM different_name; +SELECT 'S3_new', cartodb_id, another_field FROM cdb_fs_loopback.different_name; \echo '## Unregistering works' -- Deregistering the first table @@ -255,7 +256,7 @@ SELECT cartodb.CDB_Federated_Table_Unregister( -- =================================================================== \echo '## Target conflict is handled nicely: Table' -CREATE TABLE localtable (a integer); +CREATE TABLE cdb_fs_loopback.localtable (a integer); SELECT cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -265,7 +266,7 @@ SELECT cartodb.CDB_Federated_Table_Register( local_name => 'localtable'); \echo '## Target conflict is handled nicely: View' -CREATE VIEW localtable2 AS Select * from localtable; +CREATE VIEW cdb_fs_loopback.localtable2 AS Select * from cdb_fs_loopback.localtable; SELECT cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -274,8 +275,8 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'geom', local_name => 'localtable2'); -DROP VIEW localtable2; -DROP TABLE localtable; +DROP VIEW cdb_fs_loopback.localtable2; +DROP TABLE cdb_fs_loopback.localtable; -- =================================================================== -- Test permissions @@ -292,6 +293,9 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'geom', local_name => 'localtable'); +\echo '## Normal users can not write in the import schema' +CREATE TABLE cdb_fs_loopback.localtable (a integer); + \echo '## Listing remote tables does not work without permissions' Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); @@ -311,7 +315,7 @@ SELECT cartodb.CDB_Federated_Table_Register( Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); \echo '## Selecting from a registered table with granted permissions works' -Select cartodb_id, ST_AsText(the_geom) from localtable; +Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; \echo '## Selecting from a registered table without permissions does not work' \c contrib_regression cdb_fs_tester2 @@ -329,7 +333,7 @@ EXCEPTION RETURN FALSE; END $$ LANGUAGE 'plpgsql'; -Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from localtable$$); +Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable$$); DROP FUNCTION catch_permission_error(text); \echo '## Deleting a registered table without permissions does not work' @@ -347,7 +351,7 @@ SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role : SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name); \c contrib_regression cdb_fs_tester2 Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema'); -Select cartodb_id, ST_AsText(the_geom) from localtable; +Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable; \echo '## A different user can unregister a table' SELECT cartodb.CDB_Federated_Table_Unregister( diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index b8f9abc..f8dd882 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -3,24 +3,24 @@ C1| R1| S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 -t|remote_geom|public.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering another existing table works R2| S2|3|POINT(3 3)|patata -t|remote_geom|public.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|public."myFullTable"|id|geom|geom_mercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback."myFullTable"|id|geom|geom_mercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Re-registering a table works R3| -ERROR: relation "myFullTable" does not exist at character 49 +ERROR: relation "cdb_fs_loopback.myFullTable" does not exist at character 49 S3_new|3|patata ## Unregistering works U1| ERROR: relation "remote_geom" does not exist at character 71 f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering a table: Invalid server fails ERROR: Server "Does not exist" does not exist @@ -50,15 +50,17 @@ ERROR: non geometry column "Does not exists" ## Target conflict is handled nicely: Table CREATE TABLE -ERROR: Could not import table "remote_geom" as "localtable" already exists: "localtable" is not a view +ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists ## Target conflict is handled nicely: View CREATE VIEW -ERROR: Could not import table "remote_geom" as "localtable2" already exists: cannot change name of view column "a" to "cartodb_id" +ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable2" already exists DROP VIEW DROP TABLE ## Registering tables does not work without permissions You are now connected to database "contrib_regression" as user "cdb_fs_tester". ERROR: Not enough permissions to access the server "loopback" +## Normal users can not write in the import schema +ERROR: permission denied for schema cdb_fs_loopback at character 14 ## Listing remote tables does not work without permissions ERROR: Not enough permissions to access the server "loopback" ## Registering tables works with granted permissions @@ -67,8 +69,8 @@ You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester". ## Listing remote tables works with granted permissions -t|remote_geom|public.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Selecting from a registered table with granted permissions works 1|POINT(1 1) @@ -86,16 +88,16 @@ ERROR: You do not have rights to grant access on "loopback" You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester2". -t|remote_geom|public.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] +t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] 1|POINT(1 1) 2|POINT(2 2) ## A different user can unregister a table -NOTICE: drop cascades to view localtable +NOTICE: drop cascades to view cdb_fs_loopback.localtable f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|public.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Only the owner can revoke permissions over the server ERROR: You do not have rights to revoke access on "loopback" From 0893fae0257763cdf909ff64262ff38e4b58eb5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 18:46:02 +0100 Subject: [PATCH 099/108] Revert "CDB_Federated_Server_Unregister: Drop owned in cascade" This reverts commit 1b8651700710984b13f4a61ca0dfe8c9451f9117. --- scripts-available/CDB_FederatedServer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index bdf8d3a..51baa20 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -309,7 +309,7 @@ BEGIN SET client_min_messages = ERROR; BEGIN EXECUTE FORMAT ('DROP USER MAPPING FOR PUBLIC SERVER %I', server_internal); - EXECUTE FORMAT ('DROP OWNED BY %I CASCADE', role_name); + EXECUTE FORMAT ('DROP OWNED BY %I', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server; From adca72716979970cbb94f56772874c1df8545e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 18:46:45 +0100 Subject: [PATCH 100/108] Revert "Go back to public user mappings" This reverts commit 64e185b8415b91a794ff84b45028868181b74572. --- scripts-available/CDB_FederatedServer.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 51baa20..008bc4f 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -258,7 +258,7 @@ BEGIN EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name); EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name); EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name); - EXECUTE FORMAT ('CREATE USER MAPPING FOR PUBLIC SERVER %I', server_internal); + EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM USING HINT = 'Please clean the left over objects'; @@ -284,12 +284,12 @@ BEGIN LOOP IF NOT EXISTS ( WITH a AS ( - SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = 'public' + SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name ) SELECT * from a where options = option.key) THEN - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value); ELSE - EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', server_internal, option.key, option.value); + EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value); END IF; END LOOP; END @@ -308,7 +308,7 @@ DECLARE BEGIN SET client_min_messages = ERROR; BEGIN - EXECUTE FORMAT ('DROP USER MAPPING FOR PUBLIC SERVER %I', server_internal); + EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal); EXECUTE FORMAT ('DROP OWNED BY %I', role_name); EXECUTE FORMAT ('DROP ROLE %I', role_name); EXCEPTION WHEN OTHERS THEN From fa5f0bcbc12ba70cc84079a226867a93bbca22fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 18:55:38 +0100 Subject: [PATCH 101/108] CDB_Federated_Server_Grant_Access: comment about search_path --- scripts-available/CDB_FederatedServer.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 008bc4f..49a7de0 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -362,6 +362,8 @@ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; -- -- Grant access to a server +-- In the future we might consider adding the server's view schema to the role search_path +-- to make it easier to access the created views -- CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME) RETURNS void From e99e4a02c9f43b73761cd8c52a8b5965156b632b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 14 Nov 2019 19:00:09 +0100 Subject: [PATCH 102/108] Remove TODO Since the server registration requires superadmin priviledges it doesn't make sense to over protect the role creation --- scripts-available/CDB_FederatedServer.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 49a7de0..6d6fb08 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -238,10 +238,6 @@ BEGIN IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN BEGIN EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal); - -- TODO: Delete this IF before merging to make sure nobody creates a role - -- that is later used automatically by us granting them all permissions in the foreign server - -- TODO: This is here to help debugging during development (so failures to destroy objects are allowed) - -- TODO IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name); END IF; From fde12c818a77241a7f75685c2fe1ef46f247e0b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 15 Nov 2019 13:09:52 +0100 Subject: [PATCH 103/108] Update comments --- scripts-available/CDB_FederatedServer.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 6d6fb08..bf5a841 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -26,7 +26,7 @@ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- -- Produce a valid DB name for servers generated for the Federated Server -- If check_existence is true, it'll throw if the server doesn't exists --- This name is also used to +-- This name is also used as the schema to store views -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL) RETURNS NAME @@ -243,9 +243,10 @@ BEGIN END IF; EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name); + -- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases + -- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name); EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name); - EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name); EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name); EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name); From 9372c9f4e0e9f26e4fce07c00417104ef4dd34ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 5 Dec 2019 16:26:37 +0100 Subject: [PATCH 104/108] FederatedServer: Fix a bug when extracting the webmercator column from the view definition --- scripts-available/CDB_FederatedServerTables.sql | 6 +++--- test/CDB_FederatedServerTables.sql | 17 +++++++++++++++++ test/CDB_FederatedServerTables_expect | 10 ++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 013de5e..2dfe519 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -87,7 +87,7 @@ AS $$ ELSE col_def[array_length(col_def, 1) - 2] END, '.', 2)) FROM column_definitions - WHERE col_def[array_length(col_def, 1)] = 'the_geom,' + WHERE col_def[array_length(col_def, 1)] IN ('the_geom,', 'the_geom') LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; @@ -106,7 +106,7 @@ AS $$ ELSE col_def[array_length(col_def, 1) - 2] END, '.', 2)) FROM column_definitions - WHERE col_def[array_length(col_def, 1)] = 'the_geom_webmercator,' + WHERE col_def[array_length(col_def, 1)] IN ('the_geom_webmercator,', 'the_geom_webmercator') LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; @@ -218,7 +218,7 @@ BEGIN IF webmercator_column IS NULL THEN webmercator_column := geom_column; END IF; - + IF local_name IS NULL THEN local_name := remote_table; END IF; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index fdd1d7a..cedd6b1 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -52,6 +52,7 @@ INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3 CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text); INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes'); +CREATE TABLE remote_schema.remote_geom3(id bigint, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); -- =================================================================== -- Test the listing functions @@ -251,6 +252,22 @@ SELECT cartodb.CDB_Federated_Table_Unregister( remote_table => 'remote_geom' ); +\echo '##Registering a table with extra columns show the correct information' +SELECT cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom3', + id_column => 'id', + geom_column => 'geom', + webmercator_column => 'geom_mercator' + ); +Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom3'; +SELECT cartodb.CDB_Federated_Table_Unregister( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom3' + ); + -- =================================================================== -- Test conflicts -- =================================================================== diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index f8dd882..9fbeabd 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -5,12 +5,14 @@ S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering another existing table works R2| S2|3|POINT(3 3)|patata t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] t|remote_geom2|cdb_fs_loopback."myFullTable"|id|geom|geom_mercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Re-registering a table works R3| @@ -21,6 +23,7 @@ U1| ERROR: relation "remote_geom" does not exist at character 71 f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering a table: Invalid server fails ERROR: Server "Does not exist" does not exist @@ -48,6 +51,10 @@ ERROR: non geometry column "Does not exists" ## Registering a table: NULL webmercator_column is OK +##Registering a table with extra columns show the correct information + +t|remote_geom3|cdb_fs_loopback.remote_geom3|id|geom|geom_mercator|[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] + ## Target conflict is handled nicely: Table CREATE TABLE ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists @@ -71,6 +78,7 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". ## Listing remote tables works with granted permissions t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Selecting from a registered table with granted permissions works 1|POINT(1 1) @@ -90,6 +98,7 @@ You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester2". t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] 1|POINT(1 1) 2|POINT(2 2) @@ -98,6 +107,7 @@ NOTICE: drop cascades to view cdb_fs_loopback.localtable f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Only the owner can revoke permissions over the server ERROR: You do not have rights to revoke access on "loopback" From dac135890db229ad44d57e68132b0bed7146695c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 10 Dec 2019 18:08:25 +0100 Subject: [PATCH 105/108] CDB_Federated_Table_Register: Reuse webmercator_column if geom_column is NULL --- scripts-available/CDB_FederatedServerTables.sql | 4 +++- test/CDB_FederatedServerTables.sql | 10 +++++++--- test/CDB_FederatedServerTables_expect | 6 ++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 2dfe519..39b0f2c 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -214,9 +214,11 @@ BEGIN RAISE EXCEPTION 'Remote table name cannot be NULL'; END IF; - -- Use geom_column as default for webmercator_column + -- Make do with whatever columns are provided IF webmercator_column IS NULL THEN webmercator_column := geom_column; + ELSIF geom_column IS NULL THEN + geom_column := webmercator_column; END IF; IF local_name IS NULL THEN diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index cedd6b1..5941553 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -213,14 +213,17 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'Does not exists' ); -\echo '## Registering a table: NULL geom_column is OK' +\echo '## Registering a table: NULL geom_column is OK: Reuses geom_mercator' SELECT cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom', id_column => 'id', - geom_column => NULL::text + geom_column => NULL::text, + webmercator_column => 'geom' ); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; + SELECT cartodb.CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', @@ -237,7 +240,7 @@ SELECT cartodb.CDB_Federated_Table_Register( webmercator_column => 'Does not exists' ); -\echo '## Registering a table: NULL webmercator_column is OK' +\echo '## Registering a table: NULL webmercator_column is OK: Reuses geom' SELECT cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', @@ -246,6 +249,7 @@ SELECT cartodb.CDB_Federated_Table_Register( geom_column => 'geom', webmercator_column => NULL::text ); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom'; SELECT cartodb.CDB_Federated_Table_Unregister( server => 'loopback', remote_schema => 'remote_schema', diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index 9fbeabd..df646d8 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -43,13 +43,15 @@ ERROR: non integer id_column "Does not exist" ERROR: non integer id_column "" ## Registering a table: Invalid geom_column fails ERROR: non geometry column "Does not exists" -## Registering a table: NULL geom_column is OK +## Registering a table: NULL geom_column is OK: Reuses geom_mercator +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] ## Registering a table: Invalid webmercator_column fails ERROR: non geometry column "Does not exists" -## Registering a table: NULL webmercator_column is OK +## Registering a table: NULL webmercator_column is OK: Reuses geom +t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] ##Registering a table with extra columns show the correct information From 3c869e3d5a6da5e4af5dcd756731eecfa8d5fe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 11 Dec 2019 18:37:26 +0100 Subject: [PATCH 106/108] FServer list: Return the username if the user has been granted access to the server --- scripts-available/CDB_FederatedServer.sql | 33 ++++++++++++++++++++--- test/CDB_FederatedServer.sql | 3 +++ test/CDB_FederatedServer_expect | 3 +++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index bf5a841..d5cfa5e 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -192,6 +192,34 @@ END $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +-- Given an server name, returns the username used in the configuration if the caller has rights to access it +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME) +RETURNS text +AS $$ +DECLARE + role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name); + username text; +BEGIN + BEGIN + EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name); + EXCEPTION WHEN OTHERS THEN + RETURN NULL; + END; + + SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username + FROM pg_foreign_server s + LEFT JOIN pg_user_mappings u + ON u.srvid = s.oid + WHERE s.srvname = internal_server_name + ORDER BY 1; + + RESET ROLE; + + RETURN username; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + -------------------------------------------------------------------------------- -- Public functions @@ -233,7 +261,7 @@ BEGIN RAISE EXCEPTION 'postgres_fdw extension is not installed' USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`'; END IF; - + -- We only create server and roles if the server didn't exist before IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN BEGIN @@ -345,8 +373,7 @@ BEGIN (SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName", CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode", - -- Read username from pg_user_mappings - (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') AS "Username" + @extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username" FROM pg_foreign_server s LEFT JOIN pg_user_mappings u ON u.srvid = s.oid diff --git a/test/CDB_FederatedServer.sql b/test/CDB_FederatedServer.sql index 1f418c2..d314790 100644 --- a/test/CDB_FederatedServer.sql +++ b/test/CDB_FederatedServer.sql @@ -175,6 +175,9 @@ SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::te \echo '## Granting access to a user works' SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name); +\c contrib_regression cdb_fs_tester +SELECT '9.55', cartodb.CDB_Federated_Server_List_Servers(); +\c contrib_regression postgres SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name); SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name); diff --git a/test/CDB_FederatedServer_expect b/test/CDB_FederatedServer_expect index f21f2cc..836646d 100644 --- a/test/CDB_FederatedServer_expect +++ b/test/CDB_FederatedServer_expect @@ -47,6 +47,9 @@ ERROR: Could not create server myRemote4: permission denied for foreign-data wr You are now connected to database "contrib_regression" as user "postgres". ## Granting access to a user works 9.5| +You are now connected to database "contrib_regression" as user "cdb_fs_tester". +9.55|(myRemote3,postgres_fdw,localhost,5432,,read-only,fdw_user) +You are now connected to database "contrib_regression" as user "postgres". ERROR: Server "does not exist" does not exist ERROR: Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist ## Granting access again raises a notice From b3503ea42902fff907aec4f070b28100c6b523da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 12 Dec 2019 15:52:50 +0100 Subject: [PATCH 107/108] FederatedTableList: Fix column parsing with default names --- .../CDB_FederatedServerTables.sql | 38 +++++++++++-------- test/CDB_FederatedServerTables.sql | 14 +++---- test/CDB_FederatedServerTables_expect | 12 +++--- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index 39b0f2c..302b3bd 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -59,54 +59,62 @@ $$ LANGUAGE SQL STABLE PARALLEL UNSAFE; -- -- Returns the id column from a view definition +-- Note: The id is always of one of this forms: +-- SELECT t.cartodb_id, +-- SELECT t.my_id as cartodb_id, -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT) RETURNS TEXT AS $$ WITH column_definitions AS ( - SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def ) - SELECT split_part(col_def[array_length(col_def, 1) - 2], '.', 2) - FROM column_definitions where col_def[array_length(col_def, 1)] = 'cartodb_id,' + SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2)) + FROM column_definitions + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id') LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- -- Returns the geom column from a view definition -- +-- Note: The the_geom is always of one of this forms: +-- t.the_geom, +-- t.my_geom as the_geom, +-- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT) RETURNS TEXT AS $$ WITH column_definitions AS ( - SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def ) - SELECT trim(trailing ',' FROM split_part( - CASE WHEN col_def[array_length(col_def, 1) - 2] = '4326)' THEN col_def[array_length(col_def, 1) - 3] - ELSE col_def[array_length(col_def, 1) - 2] - END, '.', 2)) + SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) FROM column_definitions - WHERE col_def[array_length(col_def, 1)] IN ('the_geom,', 'the_geom') + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom') LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; -- -- Returns the webmercatorcolumn from a view definition +-- Note: The the_geom_webmercator is always of one of this forms: +-- t.the_geom_webmercator, +-- t.my_geom as the_geom_webmercator, +-- Or without the trailing comma: +-- t.the_geom_webmercator +-- t.my_geom as the_geom_webmercator -- CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT) RETURNS TEXT AS $$ WITH column_definitions AS ( - SELECT regexp_split_to_array(regexp_split_to_table(view_def, '\n'), ' ') AS col_def + SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def ) - SELECT trim(trailing ',' FROM split_part( - CASE WHEN col_def[array_length(col_def, 1) - 2] = '3857)' THEN col_def[array_length(col_def, 1) - 3] - ELSE col_def[array_length(col_def, 1) - 2] - END, '.', 2)) + SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2)) FROM column_definitions - WHERE col_def[array_length(col_def, 1)] IN ('the_geom_webmercator,', 'the_geom_webmercator') + WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator') LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index 5941553..0982398 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -45,7 +45,7 @@ CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry); INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry); -CREATE TABLE remote_schema.remote_geom2(id bigint, another_field text, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); +CREATE TABLE remote_schema.remote_geom2(cartodb_id bigint, another_field text, the_geom geometry(Geometry,4326), the_geom_webmercator geometry(Geometry,3857)); INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)'); @@ -84,9 +84,9 @@ SELECT 'R2', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom2', - id_column => 'id', - geom_column => 'geom', - webmercator_column => 'geom_mercator', + id_column => 'cartodb_id', + geom_column => 'the_geom', + webmercator_column => 'the_geom_webmercator', local_name => 'myFullTable' ); @@ -101,10 +101,10 @@ SELECT 'R3', cartodb.CDB_Federated_Table_Register( server => 'loopback', remote_schema => 'remote_schema', remote_table => 'remote_geom2', - id_column => 'id', + id_column => 'cartodb_id', -- Switch geom and geom_column on purpose to force ST_Transform to be used - geom_column => 'geom_mercator', - webmercator_column => 'geom', + geom_column => 'the_geom_webmercator', + webmercator_column => 'the_geom', local_name => 'different_name' ); diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index df646d8..6e02023 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -4,14 +4,14 @@ R1| S1|1|POINT(1 1)|patata S1|2|POINT(2 2)|patata2 t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering another existing table works R2| S2|3|POINT(3 3)|patata t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback."myFullTable"|id|geom|geom_mercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback."myFullTable"|cartodb_id|the_geom|the_geom_webmercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Re-registering a table works @@ -22,7 +22,7 @@ S3_new|3|patata U1| ERROR: relation "remote_geom" does not exist at character 71 f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Registering a table: Invalid server fails @@ -79,7 +79,7 @@ You are now connected to database "contrib_regression" as user "cdb_fs_tester". ## Listing remote tables works with granted permissions t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Selecting from a registered table with granted permissions works @@ -99,7 +99,7 @@ You are now connected to database "contrib_regression" as user "postgres". You are now connected to database "contrib_regression" as user "cdb_fs_tester2". t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] 1|POINT(1 1) @@ -108,7 +108,7 @@ f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Ty NOTICE: drop cascades to view cdb_fs_loopback.localtable f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}] -t|remote_geom2|cdb_fs_loopback.different_name|id|geom_mercator|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] +t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}] f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}] f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}] ## Only the owner can revoke permissions over the server From 6ebffe1b2c3dcbabcd058243a4fc888ced51786f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Dec 2019 11:26:56 +0100 Subject: [PATCH 108/108] PR review improvements --- scripts-available/CDB_FederatedServer.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index d5cfa5e..fabdc62 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -34,7 +34,7 @@ AS $$ DECLARE internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name); BEGIN - IF input_name IS NULL THEN + IF input_name IS NULL OR char_length(input_name) = 0 THEN RAISE EXCEPTION 'Server name cannot be NULL'; END IF; @@ -381,7 +381,7 @@ BEGIN ORDER BY 1; END $$ -LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; +LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE; --