diff --git a/scripts-available/CDB_FederatedServer.sql b/scripts-available/CDB_FederatedServer.sql index 6f0ec59..2d6375b 100644 --- a/scripts-available/CDB_FederatedServer.sql +++ b/scripts-available/CDB_FederatedServer.sql @@ -209,7 +209,7 @@ BEGIN EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', server_internal); EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM - USING HINT = 'Please clean the remaining objects"'; + USING HINT = 'Please clean the left over objects'; END; END IF; diff --git a/scripts-available/CDB_FederatedServerListRemote.sql b/scripts-available/CDB_FederatedServerListRemote.sql index e445cf2..6e66dbc 100644 --- a/scripts-available/CDB_FederatedServerListRemote.sql +++ b/scripts-available/CDB_FederatedServerListRemote.sql @@ -18,6 +18,7 @@ DECLARE inf_schema name := 'information_schema'; remote_table name := 'schemata'; local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema); + role_name text := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal); BEGIN -- Import the foreign schemata if not done IF NOT EXISTS ( @@ -82,18 +83,18 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List remote schemas in a federated server that the current user has access to. -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(remote_server name) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT) RETURNS TABLE(remote_schema name) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type WHEN 'postgres_fdw' THEN RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal); ELSE - RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server; + RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server; END CASE; END $$ @@ -102,11 +103,11 @@ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; -- -- List remote tables in a federated server that the current user has access to. -- -CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(remote_server name, remote_schema name) +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT) RETURNS TABLE(remote_table name) AS $$ DECLARE - server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := remote_server, check_existence := true); + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := true); server_type name := @extschema@.__CDB_FS_server_type(server_internal); BEGIN CASE server_type diff --git a/scripts-available/CDB_FederatedServerTables.sql b/scripts-available/CDB_FederatedServerTables.sql index e69de29..8f32b59 100644 --- a/scripts-available/CDB_FederatedServerTables.sql +++ b/scripts-available/CDB_FederatedServerTables.sql @@ -0,0 +1,173 @@ +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid IN (SELECT oid FROM pg_type + WHERE typname IN + ('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8')); + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME) +RETURNS boolean +AS $$ +BEGIN + PERFORM atttypid FROM pg_catalog.pg_attribute + WHERE attrelid = input_table + AND attname = colname + AND atttypid = 'geometry'::regtype; + RETURN FOUND; +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS) +RETURNS SETOF NAME +AS $$ + SELECT + a.attname as "colname" + FROM pg_catalog.pg_attribute a + WHERE + a.attnum > 0 + AND NOT a.attisdropped + AND a.attrelid = ( + SELECT c.oid + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = input_table::oid + ) + ORDER BY a.attnum; +$$ LANGUAGE SQL; + + + +-- +-- Set up a Federated Table +-- +-- Precondition: the federated server has to be set up via +-- CDB_SetUp_PG_Federated_Server +-- +-- Postcondition: it generates a view in the schema of the user that +-- can be used through SQL and Maps API's. +-- +-- E.g: +-- SELECT cartodb.CDB_SetUp_PG_Federated_Table( +-- 'amazon', -- mandatory, name of the federated server +-- 'my_remote_schema', -- mandatory, schema name +-- 'my_remote_table', -- mandatory, table name +-- 'id', -- mandatory, name of the id column +-- 'geom', -- optional, name of the geom column, preferably in 4326 +-- 'webmercator' -- optional, should be in 3857 if present +-- ); +-- +CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register( + server TEXT, + remote_schema TEXT, + remote_table TEXT, + id_column TEXT, + geom_column TEXT DEFAULT NULL, + webmercator_column TEXT DEFAULT NULL, + local_name NAME DEFAULT NULL +) +RETURNS void +AS $$ +DECLARE + server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name := server, check_existence := false); + local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema); + src_table REGCLASS; + + rest_of_cols TEXT[]; + geom_expression TEXT; + webmercator_expression TEXT; + carto_columns_expression TEXT[]; +BEGIN + -- Use geom_column as default for webmercator_column + IF webmercator_column IS NULL THEN + webmercator_column := geom_column; + END IF; + + IF local_name IS NULL THEN + local_name := remote_table; + END IF; + + -- Import the foreign table + EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', remote_schema, remote_table, server_internal, local_schema); + src_table := format('%I.%I', local_schema, remote_table); + + --- Grant SELECT to fdw role (TODO: Re-enable if needed) + --- EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name); + + -- Check id_column is numeric + IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN + RAISE EXCEPTION 'non integer id_column "%"', id_column; + END IF; + + -- Check if the geom and mercator columns have a geometry type (if provided) + IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN + RAISE EXCEPTION 'non geometry column "%"', geom_column; + END IF; + IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN + RAISE EXCEPTION 'non geometry column "%"', webmercator_column; + END IF; + + -- Get a list of columns excluding the id, geom and the_geom_webmercator + SELECT ARRAY( + SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c + WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL) + ) INTO rest_of_cols; + + IF geom_column IS NULL + THEN + geom_expression := 'NULL AS the_geom'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326 + THEN + geom_expression := format('t.%I AS the_geom', geom_column); + ELSE + -- It needs an ST_Transform to 4326 + geom_expression := format('@postgisschema@.ST_Transform(t.%I, 4326) AS the_geom', geom_column); + END IF; + + IF webmercator_column IS NULL + THEN + webmercator_expression := 'NULL AS the_geom_webmercator'; + ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857 + THEN + webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column); + ELSE + -- It needs an ST_Transform to 3857 + webmercator_expression := format('@postgisschema@.ST_Transform(t.%I, 3857) AS the_geom_webmercator', webmercator_column); + END IF; + + -- CARTO columns expressions + carto_columns_expression := ARRAY[ + format('t.%1$I AS cartodb_id', id_column), + geom_expression, + webmercator_expression + ]; + + -- Create a view with homogeneous CDB fields + BEGIN + EXECUTE format( + 'CREATE OR REPLACE VIEW %1$I AS + SELECT %2s + FROM %3$s t', + local_name, + array_to_string(carto_columns_expression || rest_of_cols, ','), + src_table + ); + EXCEPTION WHEN OTHERS THEN + RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM; + END; + + -- TODO: Handle this Grant perms to the view + -- EXECUTE format('GRANT SELECT ON %I TO %s', table_name, fdw_objects_name); +END +$$ +LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_FederatedServerListRemote.sql b/test/CDB_FederatedServerListRemote.sql index ff2ac36..1e0289d 100644 --- a/test/CDB_FederatedServerListRemote.sql +++ b/test/CDB_FederatedServerListRemote.sql @@ -83,10 +83,10 @@ SET client_min_messages TO notice; -- Test the listing functions -- =================================================================== \echo 'Test listing of remote schemas (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(remote_server => 'loopback'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback'); \echo 'Test listing of remote tables (sunny day)' -SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(remote_server => 'loopback', remote_schema => 'S 1'); +SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1'); -- =================================================================== -- Cleanup @@ -111,4 +111,5 @@ SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text) SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server := 'loopback2'::text); DROP DATABASE cdb_fs_tester; DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; \set QUIET off diff --git a/test/CDB_FederatedServerTables.sql b/test/CDB_FederatedServerTables.sql index e69de29..a336a20 100644 --- a/test/CDB_FederatedServerTables.sql +++ b/test/CDB_FederatedServerTables.sql @@ -0,0 +1,82 @@ +-- =================================================================== +-- create FDW objects +-- =================================================================== +\set QUIET on +SET client_min_messages TO error; +\set VERBOSITY terse +SET SESSION AUTHORIZATION postgres; +CREATE EXTENSION postgres_fdw; +CREATE ROLE cdb_fs_tester SUPERUSER LOGIN PASSWORD 'cdb_fs_passwd'; +CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester; + +SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{ + "server": { + "host": "localhost", + "port": @@PGPORT@@ + }, + "credentials": { + "username": "cdb_fs_tester", + "password": "cdb_fs_passwd" + } +}'::jsonb); + + +-- =================================================================== +-- create objects used through FDW loopback server +-- =================================================================== + +\c cdb_fs_tester postgres + +CREATE EXTENSION postgis; + +\c cdb_fs_tester cdb_fs_tester + +CREATE SCHEMA remote_schema; +CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326)); + +INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry); +INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry); + +CREATE TABLE remote_schema.remote_geom2(id bigint, another_field text, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857)); + +INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)'); + +CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text); +INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes'); + + +-- =================================================================== +-- Test the listing functions +-- =================================================================== + +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse +\set QUIET off + +SELECT 'R1', cartodb.CDB_Federated_Table_Register( + server => 'loopback', + remote_schema => 'remote_schema', + remote_table => 'remote_geom', + id_column => 'id', + geom_column => 'geom' + ); + +SELECT 'V1', pg_get_viewdef('remote_geom'); +SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom; + + +-- =================================================================== +-- Cleanup +-- =================================================================== + +\set QUIET on +\c contrib_regression postgres +SET client_min_messages TO error; +\set VERBOSITY terse + +SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text); +DROP DATABASE cdb_fs_tester; +DROP ROLE cdb_fs_tester; +DROP EXTENSION postgres_fdw; +\set QUIET off diff --git a/test/CDB_FederatedServerTables_expect b/test/CDB_FederatedServerTables_expect index e69de29..75299aa 100644 --- a/test/CDB_FederatedServerTables_expect +++ b/test/CDB_FederatedServerTables_expect @@ -0,0 +1,10 @@ +C1| +R1| +V1| SELECT t.id AS cartodb_id, + t.geom AS the_geom, + st_transform(t.geom, 3857) AS the_geom_webmercator, + t.another_field + FROM cdb_fs_schema_b904664b5208433cd85a1693ba4f7570.remote_geom t; +S1|1|POINT(1 1)|patata +S1|2|POINT(2 2)|patata2 +D1|