cartodb4/cartodb-postgresql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Revert 0.33.0"

Browse Source 
This reverts commit 24cb6cf9c1.
This commit is contained in:
Raúl Marín
2019-12-24 00:33:35 +01:00
parent 8974986d61
commit 9010642326
23 changed files with 2793 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
NEWS.md
View File

@@ -1,3 +1,6 @@
0.35.0 (XXXX-XX-XX)
* Reapply the changes in 0.33.0 (the issue we were looking for was unrelated)
0.34.0 (2019-12-23)
* Revert changes done in 0.33.0, keeping function signature to drop them

452
scripts-available/CDB_FederatedServer.sql
View File

@@ -1,15 +1,437 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Name_Pattern();
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Name(TEXT, BOOL);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Extract_Server_Name(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Schema_Name(NAME, TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Role_Name(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Create_Schema(NAME, TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_server_type(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_credentials_to_user_mapping(JSONB);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_add_default_options(jsonb);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_get_usermapping_username(NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Register_PG(TEXT, JSONB);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Unregister(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Servers(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Grant_Access(TEXT, NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Revoke_Access(TEXT, NAME);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- This function is just a placement to store and use the pattern for
-- foreign object names
-- Servers: cdb_fs_$(server_name)
-- View schema: cdb_fs_$(server_name)
-- > This is where all views created when importing tables are placed
-- > One server has only one view schema
-- Import Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name))
-- > This is where the foreign tables are placed
-- > One server has one import schema per remote schema plus auxiliar ones used
-- to access the remote catalog (pg_catalog, information_schema...)
-- Owner role: cdb_fs_$(md5sum(current_database() || server_name)
-- > This is the role than owns all schemas and tables related to the server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern()
RETURNS TEXT
AS $$
SELECT 'cdb_fs_'::text;
$$
LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid DB name for servers generated for the Federated Server
-- If check_existence is true, it'll throw if the server doesn't exists
-- This name is also used as the schema to store views
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL)
RETURNS NAME
AS $$
DECLARE
internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name);
BEGIN
IF input_name IS NULL OR char_length(input_name) = 0 THEN
RAISE EXCEPTION 'Server name cannot be NULL';
END IF;
-- We discard anything that would be truncated
IF (char_length(internal_server_name) >= 64) THEN
RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name;
END IF;
IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN
RAISE EXCEPTION 'Server "%" does not exist', input_name;
END IF;
RETURN internal_server_name::name;
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
--
-- Given the internal name for a remote server, it returns the name used by the user
-- Reverses __CDB_FS_Generate_Server_Name
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME)
RETURNS TEXT
AS $$
SELECT right(internal_server_name,
char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT;
$$
LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid name for a schema generated for the Federated Server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT)
RETURNS NAME
AS $$
DECLARE
hash_value text := md5(internal_server_name::text || '__' || schema_name::text);
BEGIN
IF schema_name IS NULL THEN
RAISE EXCEPTION 'Schema name cannot be NULL';
END IF;
RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name;
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid name for a role generated for the Federated Server
-- This needs to include the current database in its hash to avoid collisions in clusters with more than one database
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME)
RETURNS NAME
AS $$
DECLARE
hash_value text := md5(current_database()::text || '__' || internal_server_name::text);
role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value);
BEGIN
RETURN role_name::name;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL SAFE;
--
-- Creates (if not exist) a schema to place the objects for a remote schema
-- The schema is with the same AUTHORIZATION as the server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT)
RETURNS NAME
AS $$
DECLARE
schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
BEGIN
-- By changing the local role to the owner of the server we have an
-- easy way to check for permissions and keep all objects under the same owner
BEGIN
EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
EXCEPTION
WHEN invalid_parameter_value THEN
RAISE EXCEPTION 'Server "%" does not exist',
@extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
END;
IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN
EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name);
END IF;
RETURN schema_name;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Returns the type of a server by internal name
-- Currently all of them should be postgres_fdw
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME)
RETURNS name
AS $$
SELECT f.fdwname
FROM pg_foreign_server s
JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid
WHERE s.srvname = internal_server_name;
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB)
RETURNS jsonb
AS $$
DECLARE
mapping jsonb := '{}'::jsonb;
BEGIN
IF NOT (input_config ? 'credentials') THEN
RAISE EXCEPTION 'Credentials are mandatory';
END IF;
-- For now, allow not passing username or password
IF input_config->'credentials'->'username' IS NOT NULL THEN
mapping := jsonb_build_object('user', input_config->'credentials'->'username');
END IF;
IF input_config->'credentials'->'password' IS NOT NULL THEN
mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password');
END IF;
RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping);
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-- Take a config jsonb as input and return it augmented with default
-- options
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb)
RETURNS jsonb
AS $$
DECLARE
default_options jsonb := '{
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
}';
server_config jsonb;
BEGIN
IF NOT (input_config ? 'server') THEN
RAISE EXCEPTION 'Server information is mandatory';
END IF;
server_config := default_options || to_jsonb(input_config->'server');
RETURN jsonb_set(input_config, '{server}'::text[], server_config);
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-- Given an server name, returns the username used in the configuration if the caller has rights to access it
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME)
RETURNS text
AS $$
DECLARE
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
username text;
BEGIN
BEGIN
EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
EXCEPTION WHEN OTHERS THEN
RETURN NULL;
END;
SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username
FROM pg_foreign_server s
LEFT JOIN pg_user_mappings u
ON u.srvid = s.oid
WHERE s.srvname = internal_server_name
ORDER BY 1;
RESET ROLE;
RETURN username;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Registers a new PG server
--
-- Example config: '{
-- "server": {
-- "dbname": "fdw_target",
-- "host": "localhost",
-- "port": 5432,
-- "extensions": "postgis",
-- "updatable": "false",
-- "use_remote_estimate": "true",
-- "fetch_size": "1000"
-- },
-- "credentials": {
-- "username": "fdw_user",
-- "password": "foobarino"
-- }
-- }'
--
-- The configuration from __CDB_FS_add_default_options will be appended
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config));
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
row record;
option record;
BEGIN
IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
RAISE EXCEPTION 'postgres_fdw extension is not installed'
USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`';
END IF;
-- We only create server and roles if the server didn't exist before
IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN
BEGIN
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal);
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN
EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name);
END IF;
EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name);
-- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases
-- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database
EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name);
EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name);
EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name);
EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name);
EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM
USING HINT = 'Please clean the left over objects';
END;
END IF;
-- Add new options
FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p
LOOP
IF NOT EXISTS (
WITH a AS (
SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal
) SELECT * from a where options = row.key)
THEN
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value);
ELSE
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value);
END IF;
END LOOP;
-- Update user mapping settings
FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o
LOOP
IF NOT EXISTS (
WITH a AS (
SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name
) SELECT * from a where options = option.key)
THEN
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value);
ELSE
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value);
END IF;
END LOOP;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Drops a registered server and all the objects associated with it
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
SET client_min_messages = ERROR;
BEGIN
EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal);
EXECUTE FORMAT ('DROP OWNED BY %I', role_name);
EXECUTE FORMAT ('DROP ROLE %I', role_name);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List registered servers
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%')
RETURNS TABLE (
name text,
driver text,
host text,
port text,
dbname text,
readmode text,
username text
)
AS $$
DECLARE
server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server);
BEGIN
RETURN QUERY SELECT
-- Name as shown to the user
@extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name",
-- Which driver are we using (postgres_fdw, odbc_fdw...)
@extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver",
-- Read options from pg_foreign_server
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host",
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port",
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName",
CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode",
@extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username"
FROM pg_foreign_server s
LEFT JOIN pg_user_mappings u
ON u.srvid = s.oid
WHERE s.srvname ILIKE server_name
ORDER BY 1;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE;
--
-- Grant access to a server
-- In the future we might consider adding the server's view schema to the role search_path
-- to make it easier to access the created views
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
IF (db_role IS NULL) THEN
RAISE EXCEPTION 'User role "%" cannot be NULL', username;
END IF;
BEGIN
EXECUTE format('GRANT %I TO %I', server_role_name, db_role);
EXCEPTION
WHEN insufficient_privilege THEN
RAISE EXCEPTION 'You do not have rights to grant access on "%"', server;
WHEN OTHERS THEN
RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Revoke access to a server
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
IF (db_role IS NULL) THEN
RAISE EXCEPTION 'User role "%" cannot be NULL', username;
END IF;
BEGIN
EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role);
EXCEPTION
WHEN insufficient_privilege THEN
RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server;
WHEN OTHERS THEN
RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

252
scripts-available/CDB_FederatedServerDiagnostics.sql
View File

@@ -1,9 +1,243 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Import_If_Not_Exists(name, name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Version_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Options_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Host_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Port_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(name, float, integer);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Server_Diagnostics_PG(name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Diagnostics(TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- Import a foreign table if it does not exist
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name)
RETURNS void
AS $$
DECLARE
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
remote_schema, remote_table, server_internal, local_schema);
END IF;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the version of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name)
RETURNS text
AS $$
DECLARE
remote_schema name := 'pg_catalog';
remote_table name := 'pg_settings';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
remote_server_version text;
BEGIN
PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
BEGIN
EXECUTE format('
SELECT setting FROM %I.%I WHERE name = ''server_version'';
', local_schema, remote_table) INTO remote_server_version;
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
RETURN remote_server_version;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the PostGIS extension version of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name)
RETURNS text
AS $$
DECLARE
remote_schema name := 'pg_catalog';
remote_table name := 'pg_extension';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
remote_postgis_version text;
BEGIN
PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
BEGIN
EXECUTE format('
SELECT extversion FROM %I.%I WHERE extname = ''postgis'';
', local_schema, remote_table) INTO remote_postgis_version;
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
RETURN remote_postgis_version;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the foreign server options of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name)
RETURNS jsonb
AS $$
-- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html
-- See https://www.postgresql.org/docs/current/functions-info.html
SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server
WHERE srvname = server_internal
) AS opt;
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get a foreign PG server hostname from the catalog
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name)
RETURNS text
AS $$
SELECT option_value FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server WHERE srvname = server_internal
) AS opt WHERE opt.option_name = 'host';
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get a foreign PG server port from the catalog
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name)
RETURNS integer
AS $$
SELECT option_value::integer FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server WHERE srvname = server_internal
) AS opt WHERE opt.option_name = 'port';
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get one measure of network latency in ms to a remote TCP server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(
server_internal name,
timeout_seconds float DEFAULT 5.0,
n_samples integer DEFAULT 10
)
RETURNS jsonb
AS $$
import socket
import json
import math
from timeit import default_timer as timer
plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name'])
rv = plpy.execute(plan, [server_internal], 1)
host = rv[0]['host']
plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name'])
rv = plpy.execute(plan, [server_internal], 1)
port = rv[0]['port'] or 5432
n_errors = 0
samples = []
for i in range(n_samples):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout_seconds)
t_start = timer()
try:
s.connect((host, int(port)))
t_stop = timer()
s.shutdown(socket.SHUT_RD)
except (socket.timeout, OSError, socket.error) as ex:
plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex)))
n_errors += 1
finally:
s.close()
t_connect = (t_stop - t_start) * 1000.0
plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect))
samples.append(t_connect)
stats = {
'n_samples': n_samples,
'n_errors': n_errors,
}
n = len(samples)
if n >= 1:
mean = sum(samples) / n
stats.update({
'avg': round(mean, 3),
'min': round(min(samples), 3),
'max': round(max(samples), 3)
})
if n >= 2:
var = sum([ (x - mean)**2 for x in samples ]) / (n-1)
stdev = math.sqrt(var)
stats.update({
'stdev': round(stdev, 3)
})
return json.dumps(stats)
$$
LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
--
-- Collect and return diagnostics info from a remote PG into a jsonb
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name)
RETURNS jsonb
AS $$
DECLARE
remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal);
remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal);
remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal);
remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal);
BEGIN
RETURN jsonb_build_object(
'server_version', remote_server_version,
'postgis_version', remote_postgis_version,
'server_options', remote_server_options,
'server_latency_ms', remote_server_latency_ms
);
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Collect and return diagnostics info from a remote PG into a jsonb
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT)
RETURNS jsonb
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal);
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

305
scripts-available/CDB_FederatedServerListRemote.sql
View File

@@ -1,7 +1,298 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Schemas_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Tables_PG(name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Columns_PG(name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(name, name, name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Schemas(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Tables(TEXT, TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Columns(TEXT, TEXT, TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- List the schemas of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name)
RETURNS TABLE(remote_schema name)
AS $$
DECLARE
-- Import schemata from the information schema
--
-- "The view schemata contains all schemas in the current database
-- that the current user has access to (by way of being the owner
-- or having some privilege)."
-- See https://www.postgresql.org/docs/11/infoschema-schemata.html
--
-- "The information schema is defined in the SQL standard and can
-- therefore be expected to be portable and remain stable"
-- See https://www.postgresql.org/docs/11/information-schema.html
inf_schema name := 'information_schema';
remote_table name := 'schemata';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign schemata table
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_table, server_internal, local_schema);
END IF;
-- Return the result we're interested in. Exclude toast and temp schemas
BEGIN
RETURN QUERY EXECUTE format('
SELECT schema_name::name AS remote_schema FROM %I.%I
WHERE schema_name NOT LIKE %s
ORDER BY remote_schema
', local_schema, remote_table, '''pg_%''');
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the names of the tables in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name)
RETURNS TABLE(remote_table name)
AS $func$
DECLARE
-- Import `tables` from the information schema
--
-- "The view tables contains all tables and views defined in the
-- current database. Only those tables and views are shown that
-- the current user has access to (by way of being the owner or
-- having some privilege)."
-- https://www.postgresql.org/docs/11/infoschema-tables.html
-- Create local target schema if it does not exists
inf_schema name := 'information_schema';
remote_table name := 'tables';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign `tables` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_table, server_internal, local_schema);
END IF;
-- Note: in this context, schema names are not to be quoted
RETURN QUERY EXECUTE format($q$
SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name
$q$, local_schema, remote_table, remote_schema);
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the columns in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name)
RETURNS TABLE(table_name name, column_name name, column_type text)
AS $func$
DECLARE
-- Import `columns` from the information schema
--
-- "The view columns contains information about all table columns (or view columns)
-- in the database. System columns (oid, etc.) are not included. Only those columns
-- are shown that the current user has access to (by way of being the owner or having some privilege)."
-- https://www.postgresql.org/docs/11/infoschema-columns.html
-- Create local target schema if it does not exists
inf_schema name := 'information_schema';
remote_col_table name := 'columns';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign `columns` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_col_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_col_table, server_internal, local_schema);
END IF;
-- Note: in this context, schema names are not to be quoted
-- We join with the geometry columns to change the type `USER-DEFINED`
-- by its appropiate geometry and srid
RETURN QUERY EXECUTE format($q$
SELECT
a.table_name::name,
a.column_name::name,
COALESCE(b.column_type, a.data_type)::TEXT as column_type
FROM
%I.%I a
LEFT JOIN
@extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b
ON a.table_name = b.table_name AND a.column_name = b.column_name
WHERE table_schema = '%s'
ORDER BY a.table_name, a.column_name $q$,
local_schema, remote_col_table,
server_internal, remote_schema,
remote_schema);
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the geometry columns in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public')
RETURNS TABLE(table_name name, column_name name, column_type text)
AS $func$
DECLARE
-- Import `geometry_columns` and `geography_columns` from the postgis schema
-- We assume that postgis is installed in the public schema
-- Create local target schema if it does not exists
remote_geometry_view name := 'geometry_columns';
remote_geography_view name := 'geography_columns';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema);
BEGIN
-- Import the foreign `geometry_columns` and `geography_columns` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_geometry_view
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I',
postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema);
END IF;
BEGIN
-- Note: We return both the type and srid as the type
RETURN QUERY EXECUTE format($q$
SELECT
f_table_name::NAME as table_name,
f_geometry_column::NAME as column_name,
type::TEXT || ',' || srid::TEXT as column_type
FROM
(
SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I
) _geo_views
WHERE f_table_schema = '%s'
$q$,
local_schema, remote_geometry_view,
local_schema, remote_geography_view,
remote_schema);
EXCEPTION WHEN OTHERS THEN
RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"',
postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
RETURN;
END;
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- List remote schemas in a federated server that the current user has access to.
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT)
RETURNS TABLE(remote_schema name)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal);
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List remote tables in a federated server that the current user has access to.
-- For registered tables it returns also the associated configuration
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT)
RETURNS TABLE(
registered boolean,
remote_table TEXT,
local_qualified_name TEXT,
id_column_name TEXT,
geom_column_name TEXT,
webmercator_column_name TEXT,
columns JSON
)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY
SELECT
coalesce(registered_tables.registered, false)::boolean as registered,
foreign_tables.remote_table::text as remote_table,
registered_tables.local_qualified_name as local_qualified_name,
registered_tables.id_column_name as id_column_name,
registered_tables.geom_column_name as geom_column_name,
registered_tables.webmercator_column_name as webmercator_column_name,
remote_columns.columns as columns
FROM
@extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables
LEFT JOIN
@extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables
ON foreign_tables.remote_table = registered_tables.remote_table
LEFT JOIN
( -- Extract and group columns with their remote table
SELECT table_name,
json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns
FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
GROUP BY table_name
) remote_columns
ON foreign_tables.remote_table = remote_columns.table_name
ORDER BY foreign_tables.remote_table;
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the columns of a remote table in a federated server that the current user has access to.
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns(
server TEXT,
remote_schema TEXT,
remote_table TEXT)
RETURNS TABLE(column_n name, column_t text)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
IF remote_table IS NULL THEN
RAISE EXCEPTION 'Remote table name cannot be NULL';
END IF;
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY
SELECT
column_name,
column_type
FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
WHERE table_name = remote_table
ORDER BY column_name;
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

354
scripts-available/CDB_FederatedServerTables.sql
View File

@@ -1,9 +1,345 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Integer(REGCLASS, NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Geometry(REGCLASS, NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_GetColumns(REGCLASS);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_id_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_geom_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_webmercator_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Registered_Tables(NAME,TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Register(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Unregister(TEXT, TEXT, TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- Checks if a column is of integer type
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME)
RETURNS boolean
AS $$
BEGIN
PERFORM atttypid FROM pg_catalog.pg_attribute
WHERE attrelid = input_table
AND attname = colname
AND atttypid IN (SELECT oid FROM pg_type
WHERE typname IN
('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8'));
RETURN FOUND;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
--
-- Checks if a column is of geometry type
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME)
RETURNS boolean
AS $$
BEGIN
PERFORM atttypid FROM pg_catalog.pg_attribute
WHERE attrelid = input_table
AND attname = colname
AND atttypid = '@postgisschema@.geometry'::regtype;
RETURN FOUND;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
--
-- Returns the name of all the columns from a table
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS)
RETURNS SETOF NAME
AS $$
SELECT
a.attname as "colname"
FROM pg_catalog.pg_attribute a
WHERE
a.attnum > 0
AND NOT a.attisdropped
AND a.attrelid = (
SELECT c.oid
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.oid = input_table::oid
)
ORDER BY a.attnum;
$$ LANGUAGE SQL STABLE PARALLEL UNSAFE;
--
-- Returns the id column from a view definition
-- Note: The id is always of one of this forms:
-- SELECT t.cartodb_id,
-- SELECT t.my_id as cartodb_id,
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Returns the geom column from a view definition
--
-- Note: The the_geom is always of one of this forms:
-- t.the_geom,
-- t.my_geom as the_geom,
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Returns the webmercatorcolumn from a view definition
-- Note: The the_geom_webmercator is always of one of this forms:
-- t.the_geom_webmercator,
-- t.my_geom as the_geom_webmercator,
-- Or without the trailing comma:
-- t.the_geom_webmercator
-- t.my_geom as the_geom_webmercator
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- List all registered tables in a server + schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables(
server_internal NAME,
remote_schema TEXT
)
RETURNS TABLE(
registered boolean,
remote_table TEXT,
local_qualified_name TEXT,
id_column_name TEXT,
geom_column_name TEXT,
webmercator_column_name TEXT
)
AS $$
DECLARE
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
RETURN QUERY SELECT
true as registered,
source_table::text as remote_table,
format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name,
@extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name,
@extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name,
@extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name
FROM
(
SELECT DISTINCT
dependent_ns.nspname as dependent_schema,
dependent_view.relname as dependent_view,
source_table.relname as source_table,
pg_get_viewdef(dependent_view.oid) as view_definition
FROM pg_depend
JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace
JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
WHERE
source_ns.nspname = local_schema
ORDER BY 1,2
) _aux;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Sets up a Federated Table
--
-- Precondition: the federated server has to be set up via
-- CDB_Federated_Server_Register_PG
--
-- Postcondition: it generates a view in the schema of the user that
-- can be used through SQL and Maps API's.
-- If the table was already exported, it will be dropped and re-imported
--
-- The view is placed under the server's view schema (cdb_fs_$(server_name))
--
-- E.g:
-- SELECT cartodb.CDB_SetUp_PG_Federated_Table(
-- 'amazon', -- mandatory, name of the federated server
-- 'my_remote_schema', -- mandatory, schema name
-- 'my_remote_table', -- mandatory, table name
-- 'id', -- mandatory, name of the id column
-- 'geom', -- optional, name of the geom column, preferably in 4326
-- 'webmercator' -- optional, should be in 3857 if present
-- 'local_name' -- optional, name of the local view (uses the remote_name if not declared)
-- );
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register(
server TEXT,
remote_schema TEXT,
remote_table TEXT,
id_column TEXT,
geom_column TEXT DEFAULT NULL,
webmercator_column TEXT DEFAULT NULL,
local_name NAME DEFAULT NULL
)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
src_table REGCLASS; -- import_schema.remote_table - import_schema is local_schema
local_view text; -- view_schema.local_name - view_schema is server_internal
rest_of_cols TEXT[];
geom_expression TEXT;
webmercator_expression TEXT;
carto_columns_expression TEXT[];
BEGIN
IF remote_table IS NULL THEN
RAISE EXCEPTION 'Remote table name cannot be NULL';
END IF;
-- Make do with whatever columns are provided
IF webmercator_column IS NULL THEN
webmercator_column := geom_column;
ELSIF geom_column IS NULL THEN
geom_column := webmercator_column;
END IF;
IF local_name IS NULL THEN
local_name := remote_table;
END IF;
-- Import the foreign table
-- Drop the old view / table if there was one
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN
EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table);
END IF;
BEGIN
EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;',
remote_schema, remote_table, server_internal, local_schema);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM;
END;
BEGIN
src_table := format('%I.%I', local_schema, remote_table);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server;
END;
-- Check id_column is numeric
IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN
RAISE EXCEPTION 'non integer id_column "%"', id_column;
END IF;
-- Check if the geom and mercator columns have a geometry type (if provided)
IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN
RAISE EXCEPTION 'non geometry column "%"', geom_column;
END IF;
IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN
RAISE EXCEPTION 'non geometry column "%"', webmercator_column;
END IF;
-- Get a list of columns excluding the id, geom and the_geom_webmercator
SELECT ARRAY(
SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c
WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL)
) INTO rest_of_cols;
IF geom_column IS NULL
THEN
geom_expression := 'NULL AS the_geom';
ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326
THEN
geom_expression := format('t.%I AS the_geom', geom_column);
ELSE
-- It needs an ST_Transform to 4326
geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column);
END IF;
IF webmercator_column IS NULL
THEN
webmercator_expression := 'NULL AS the_geom_webmercator';
ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857
THEN
webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column);
ELSE
-- It needs an ST_Transform to 3857
webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column);
END IF;
-- CARTO columns expressions
carto_columns_expression := ARRAY[
format('t.%1$I AS cartodb_id', id_column),
geom_expression,
webmercator_expression
];
-- Create view schema if it doesn't exist
IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN
EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name);
END IF;
-- Create a view with homogeneous CDB fields
BEGIN
EXECUTE format(
'CREATE OR REPLACE VIEW %1$I.%2$I AS
SELECT %3s
FROM %4$s t',
server_internal, local_name,
array_to_string(carto_columns_expression || rest_of_cols, ','),
src_table
);
EXCEPTION WHEN OTHERS THEN
IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN
RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name;
ELSE
RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM;
END IF;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Unregisters a remote table. Any dependent object will be dropped
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister(
server TEXT,
remote_schema TEXT,
remote_table TEXT
)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table);
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

179
scripts-available/CDB_ForeignTable.sql
View File

@@ -139,176 +139,6 @@ $$
LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- Produce a valid DB name for objects created for the user FDW's
CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME)
RETURNS NAME AS $$
-- Note on input we use %s and on output we use %I, in order to
-- avoid double escaping
SELECT format('cdb_fdw_%s', fdw_input_name)::NAME;
$$
LANGUAGE sql IMMUTABLE PARALLEL SAFE;
-- A function to set up a user-defined foreign data server
-- It does not read from CDB_Conf.
-- Only superuser roles can invoke it successfully
--
-- Sample call:
-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{
-- "server": {
-- "extensions": "postgis",
-- "dbname": "testdb",
-- "host": "myhostname.us-east-2.rds.amazonaws.com",
-- "port": "5432"
-- },
-- "user_mapping": {
-- "user": "fdw_user",
-- "password": "secret"
-- }
-- }');
--
-- Underneath it will:
-- * Set up postgresql_fdw
-- * Create a server with the name 'cdb_fdw_amazon'
-- * Create a role called 'cdb_fdw_amazon' to manage access
-- * Create a user mapping with that role 'cdb_fdw_amazon'
-- * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign
-- tables over there
--
-- It is the responsibility of the superuser to grant that role to either:
-- * Nobody
-- * Specific roles: GRANT amazon TO role_name;
-- * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon');
-- * The publicuser: GRANT cdb_fdw_amazon TO publicuser;
CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json)
RETURNS void AS $$
DECLARE
row record;
option record;
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
BEGIN
-- TODO: refactor with original function
-- This function tries to be as idempotent as possible, by not creating anything more than once
-- (not even using IF NOT EXIST to avoid throwing warnings)
IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
CREATE EXTENSION postgres_fdw;
RAISE NOTICE 'Created postgres_fdw EXTENSION';
END IF;
-- Create FDW first if it does not exist
IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name)
THEN
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name);
RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name;
END IF;
-- Set FDW settings
FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key)
THEN
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value);
ELSE
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value);
END IF;
END LOOP;
-- Create specific role for this
IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN
EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name);
RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name;
END IF;
-- Transfer ownership of the server to the fdw role
EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
-- Create user mapping
-- NOTE: we use a PUBLIC user mapping but control access to the SERVER
-- so that we don't need to create a mapping for every user nor store credentials elsewhere
IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN
EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name);
RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name;
END IF;
-- Update user mapping settings
FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN
EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value);
ELSE
EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value);
END IF;
END LOOP;
-- Grant usage on the wrapper and server to the fdw role
EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name);
RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name;
EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name);
RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name;
-- Create schema if it does not exist.
IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN
EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name);
RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name;
END IF;
-- Give the fdw role ownership over the schema
EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name;
-- TODO: Bring here the remote cdb_tablemetadata
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- A function to drop a user-defined foreign server and all related objects
-- It does not read from CDB_Conf
-- It must be executed with a superuser role to succeed
--
-- Sample call:
-- SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon')
--
-- Note: if there's any dependent object (i.e. foreign table) this call will fail
CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false)
RETURNS void AS $$
DECLARE
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
cascade_clause NAME;
BEGIN
CASE force
WHEN true THEN
cascade_clause := 'CASCADE';
ELSE
cascade_clause := 'RESTRICT';
END CASE;
EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Dropped schema %', fdw_objects_name;
EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name);
RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name;
EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Dropped foreign server %', fdw_objects_name;
EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name;
EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name);
RAISE NOTICE 'Dropped role %', fdw_objects_name;
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- Set up a user foreign table
-- E.g:
-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable');
-- SELECT * FROM amazon.my_table;
CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME)
RETURNS void AS $$
DECLARE
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
BEGIN
EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name);
--- Grant SELECT to fdw role
EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name);
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid)
RETURNS TEXT AS $$
SELECT option_value FROM pg_options_to_table((
@@ -374,3 +204,12 @@ RETURNS timestamptz AS $$
LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
) SELECT max(updated_at) FROM t_updated_at;
$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Deprecated
--------------------------------------------------------------------------------
DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name);
DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json);
DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean);
DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name);

1
scripts-available/CDB_Groups_API.sql
View File

@@ -5,7 +5,6 @@
-- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
----------------------------------
-- TODO: delete this development cleanup before final merge
DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text);
DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text);
DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text);

26
scripts-available/CDB_Organizations.sql
View File

@@ -172,27 +172,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Role management
-- Deprecated
--------------------------------------------------------------------------------
CREATE OR REPLACE
FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name)
RETURNS VOID AS $$
DECLARE
org_role TEXT;
BEGIN
org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
EXECUTE format('GRANT %I TO %I', role_name, org_role);
END
$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE
FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name)
RETURNS VOID AS $$
DECLARE
org_role TEXT;
BEGIN
org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
EXECUTE format('REVOKE %I FROM %I', role_name, org_role);
END
$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name);

2
sql/test_setup.sql
View File

@@ -2,7 +2,7 @@
\set QUIET on
SET client_min_messages TO error;
CREATE EXTENSION postgis;
CREATE EXTENSION plpythonu;
CREATE EXTENSION @@plpythonu@@;
CREATE SCHEMA cartodb;
\i 'cartodb--unpackaged--@@VERSION@@.sql'
CREATE FUNCTION public.cdb_invalidate_varnish(table_name text)

4
test/CDB_CartodbfyTableTest.sql
View File

@@ -134,9 +134,9 @@ DROP TABLE t;
-- table with single non-geometrical column
CREATE TABLE t AS SELECT ST_SetSRID(ST_MakePoint(-1,-1),4326) as the_geom, 1::int as cartodb_id, 'this is a sentence' as description;
SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
SELECT * FROM t;
SELECT cartodb_id, the_geom, description FROM t;
SELECT CDB_CartodbfyTableCheck('t', 'check function idempotence');
SELECT * FROM t;
SELECT cartodb_id, the_geom, description FROM t;
DROP TABLE t;
-- table with existing srid-unconstrained (but type-constrained) the_geom

4
test/CDB_CartodbfyTableTest_expect
View File

@@ -7,9 +7,9 @@ single non-geometrical column cartodbfied fine
DROP TABLE
SELECT 1
check function idempotence cartodbfied fine
1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence
check function idempotence cartodbfied fine
1|0101000020E6100000000000000000F0BF000000000000F0BF|0101000020110F0000DB0B4ADA772DFBC077432E49D22DFBC0|this is a sentence
1|0101000020E6100000000000000000F0BF000000000000F0BF|this is a sentence
DROP TABLE
SELECT 1
srid-unconstrained the_geom cartodbfied fine

220
test/CDB_FederatedServer.sql Normal file
View File

@@ -0,0 +1,220 @@
-- Setup
\set QUIET on
SET client_min_messages TO error;
\set VERBOSITY terse
SET SESSION AUTHORIZATION postgres;
CREATE EXTENSION postgres_fdw;
\set QUIET off
\echo '## List empty servers shows nothing'
SELECT '1.1', cartodb.CDB_Federated_Server_List_Servers();
\echo '## List non-existent server shows nothing'
SELECT '1.2', cartodb.CDB_Federated_Server_List_Servers(server => 'doesNotExist');
\echo '## Create and list a server works'
SELECT '1.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "fdw_user",
"password": "foobarino"
}
}'::jsonb);
SELECT '1.4', cartodb.CDB_Federated_Server_List_Servers();
\echo '## Create and list a second server works'
SELECT '2.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{
"server": {
"dbname": "fdw_target",
"host": "localhost",
"port": @@PGPORT@@,
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
},
"credentials": {
"username": "fdw_user",
"password": "foobarino"
}
}'::jsonb);
SELECT '2.2', cartodb.CDB_Federated_Server_List_Servers();
\echo '## List server by name works'
SELECT '2.3', cartodb.CDB_Federated_Server_List_Servers(server => 'myRemote');
\echo '## Re-register a second server works'
SELECT '3.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote2'::text, config => '{
"server": {
"dbname": "fdw_target",
"host": "localhost",
"port": @@PGPORT@@,
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
},
"credentials": {
"username": "other_remote_user",
"password": "foobarino"
}
}'::jsonb);
SELECT '3.2', cartodb.CDB_Federated_Server_List_Servers();
\echo '## Unregister server 1 works'
SELECT '4.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote'::text);
SELECT '4.2', cartodb.CDB_Federated_Server_List_Servers();
\echo '## Unregistering a server that does not exist fails'
SELECT '5.1', cartodb.CDB_Federated_Server_Unregister(server => 'doesNotExist'::text);
\echo '## Unregister the second server works'
SELECT '6.1', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote2'::text);
SELECT '6.2', cartodb.CDB_Federated_Server_List_Servers();
\echo '## Create a server with NULL name fails'
SELECT '7.0', cartodb.CDB_Federated_Server_Register_PG(server => NULL::text, config => '{ "server": {}, "credentials" : {}}');
\echo '## Create a server with NULL config fails'
SELECT '7.01', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => NULL::jsonb);
\echo '## Create a server with empty config fails'
SELECT '7.1', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{}');
\echo '## Create a server without credentials fails'
SELECT '7.2', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
"server": {
"dbname": "fdw_target",
"host": "localhost",
"port": @@PGPORT@@,
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
}
}'::jsonb);
\echo '## Create a server with empty credentials works'
SELECT '7.3', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
"server": {
"dbname": "fdw_target",
"host": "localhost",
"port": @@PGPORT@@,
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
},
"credentials": { }
}'::jsonb);
SELECT '7.4', cartodb.CDB_Federated_Server_List_Servers();
SELECT '7.5', cartodb.CDB_Federated_Server_Unregister(server => 'empty'::text);
\echo '## Create a server without options fails'
SELECT '7.6', cartodb.CDB_Federated_Server_Register_PG(server => 'empty'::text, config => '{
"credentials": {
"username": "other_remote_user",
"password": "foobarino"
}
}'::jsonb);
\echo '## Create a server with special characters works'
SELECT '8.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote" or''not'::text, config => '{
"server": {
"dbname": "fdw target",
"host": "localhost",
"port": @@PGPORT@@,
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
},
"credentials": {
"username": "fdw user",
"password": "foo barino"
}
}'::jsonb);
SELECT '8.2', cartodb.CDB_Federated_Server_List_Servers();
SELECT '8.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote" or''not'::text);
-- Test permissions
\set QUIET on
CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
\set QUIET off
SELECT '9.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote3'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "fdw_user",
"password": "foobarino"
}
}'::jsonb);
\c contrib_regression cdb_fs_tester
\echo '## All users are able to list servers'
SELECT '9.2', cartodb.CDB_Federated_Server_List_Servers();
\echo '## Only superadmins can create servers'
SELECT '9.3', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "fdw_user",
"password": "foobarino"
}
}'::jsonb);
\c contrib_regression postgres
\echo '## Granting access to a user works'
SELECT '9.5', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT '9.55', cartodb.CDB_Federated_Server_List_Servers();
\c contrib_regression postgres
SELECT '9.6', cartodb.CDB_Federated_Server_Grant_Access(server => 'does not exist', db_role => 'cdb_fs_tester'::name);
SELECT '9.7', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'does not exist'::name);
\echo '## Granting access again raises a notice'
SELECT '9.8', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
\echo '## Revoking access to a user works'
SELECT '9.9', cartodb.CDB_Federated_Server_Revoke_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
SELECT '9.10', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote3', db_role => 'cdb_fs_tester'::name);
\echo '## Unregistering a server with active grants works'
SELECT '9.11', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote3'::text);
\echo '## A user with granted access can not drop a server'
SELECT '10.1', cartodb.CDB_Federated_Server_Register_PG(server => 'myRemote4'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "fdw_user",
"password": "foobarino"
}
}'::jsonb);
SELECT '10.2', cartodb.CDB_Federated_Server_Grant_Access(server => 'myRemote4', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT '10.3', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text);
\c contrib_regression postgres
SELECT '10.4', cartodb.CDB_Federated_Server_Unregister(server => 'myRemote4'::text);
-- Cleanup
\set QUIET on
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
DROP ROLE cdb_fs_tester;
DROP EXTENSION postgres_fdw;
\set QUIET off

125
test/CDB_FederatedServerDiagnostics.sql Normal file
View File

@@ -0,0 +1,125 @@
-- ===================================================================
-- create FDW objects
-- ===================================================================
\set QUIET on
SET client_min_messages TO error;
\set VERBOSITY terse
CREATE EXTENSION postgres_fdw;
CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
-- Create database to be used as remote
CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'wrong-port'::text, config => '{
"server": {
"host": "localhost",
"port": "12345"
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
SELECT 'C3', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback-no-port'::text, config => '{
"server": {
"host": "localhost"
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
\c cdb_fs_tester postgres
CREATE EXTENSION postgis;
\c contrib_regression postgres
\set QUIET off
-- ===================================================================
-- Test server diagnostics function(s)
-- ===================================================================
\echo '%% It raises an error if the server does not exist'
SELECT '1.1', cartodb.CDB_Federated_Server_Diagnostics(server => 'doesNotExist');
\echo '%% It returns a jsonb object'
SELECT '1.2', pg_typeof(cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback'));
\echo '%% It returns the server version'
SELECT '1.3', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"server_version": "%s"}', setting)::jsonb
FROM pg_settings WHERE name = 'server_version';
\echo '%% It returns the postgis version'
SELECT '1.4', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> format('{"postgis_version": "%s"}', extversion)::jsonb
FROM pg_extension WHERE extname = 'postgis';
\echo '%% It returns null as the postgis version if it is not installed'
\set QUIET on
\c cdb_fs_tester postgres
DROP EXTENSION postgis;
\c contrib_regression postgres
\set QUIET off
SELECT '1.5', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"postgis_version": null}'::jsonb;
\echo '%% It returns the remote server options'
SELECT '1.6', cartodb.CDB_Federated_Server_Diagnostics(server => 'loopback') @> '{"server_options": {"host": "localhost", "port": "@@PGPORT@@", "updatable": "false", "extensions": "postgis", "fetch_size": "1000", "use_remote_estimate": "true"}}'::jsonb;
\echo '%% It returns network latency stats to the remote server: min <= avg <= max'
WITH latency AS (
SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
) SELECT '2.1', (latency.ms->'min')::text::float <= (latency.ms->'avg')::text::float, (latency.ms->'avg')::text::float <= (latency.ms->'max')::text::float
FROM latency;
\echo '%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)'
WITH latency AS (
SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
) SELECT '2.2', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
FROM latency;
\echo '%% Latency stats: stdev > 0'
WITH latency AS (
SELECT CDB_Federated_Server_Diagnostics('loopback')->'server_latency_ms' ms
) SELECT '2.3', (latency.ms->'stdev')::text::float >= 0.0
FROM latency;
\echo '%% It raises an error if the wrong port is provided'
SELECT '3.0', cartodb.CDB_Federated_Server_Diagnostics(server => 'wrong-port');
\echo '%% Latency stats: can get them on default PG port 5432 when not provided'
WITH latency AS (
SELECT CDB_Federated_Server_Diagnostics('loopback-no-port')->'server_latency_ms' ms
) SELECT '2.4', 0.0 <= (latency.ms->'min')::text::float, (latency.ms->'max')::text::float <= 1000.0
FROM latency;
-- ===================================================================
-- Cleanup
-- ===================================================================
\set QUIET on
SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text);
SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'wrong-port'::text);
SELECT 'D3', cartodb.CDB_Federated_Server_Unregister(server => 'loopback-no-port'::text);
-- Reconnect, using a new session in order to close FDW connections
\connect
DROP DATABASE cdb_fs_tester;
-- Drop role
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
DROP ROLE cdb_fs_tester;
DROP EXTENSION postgres_fdw;
\set QUIET off

28
test/CDB_FederatedServerDiagnostics_expect Normal file
View File

@@ -0,0 +1,28 @@
C1|
C2|
C3|
%% It raises an error if the server does not exist
ERROR: Server "doesNotExist" does not exist
%% It returns a jsonb object
1.2|jsonb
%% It returns the server version
1.3|t
%% It returns the postgis version
1.4|t
%% It returns null as the postgis version if it is not installed
1.5|t
%% It returns the remote server options
1.6|t
%% It returns network latency stats to the remote server: min <= avg <= max
2.1|t|t
%% Latency stats: 0 <= min <= max <= 1000 ms (local connection)
2.2|t|t
%% Latency stats: stdev > 0
2.3|t
%% It raises an error if the wrong port is provided
ERROR: could not connect to server "cdb_fs_wrong-port"
%% Latency stats: can get them on default PG port 5432 when not provided
2.4|t|t
D1|
D2|
D3|

319
test/CDB_FederatedServerListRemote.sql Normal file
View File

@@ -0,0 +1,319 @@
-- ===================================================================
-- create FDW objects
-- ===================================================================
\set QUIET on
SET client_min_messages TO error;
\set VERBOSITY terse
CREATE EXTENSION postgres_fdw;
CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2;
-- Create database to be used as remote
CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
SELECT 'C2', cartodb.CDB_Federated_Server_Register_PG(server => 'loopback2'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
-- ===================================================================
-- Setup 1
-- ===================================================================
\c cdb_fs_tester cdb_fs_tester
CREATE TYPE user_enum AS ENUM ('foo', 'bar', 'buz');
CREATE SCHEMA "S 1";
CREATE TABLE "S 1"."T 1" (
"C 1" int NOT NULL,
c2 int NOT NULL,
c3 text,
c4 timestamptz,
c5 timestamp,
c6 varchar(10),
c7 char(10),
c8 user_enum,
CONSTRAINT t1_pkey PRIMARY KEY ("C 1")
);
CREATE TABLE "S 1"."T 2" (
c1 int NOT NULL,
c2 text,
CONSTRAINT t2_pkey PRIMARY KEY (c1)
);
CREATE TABLE "S 1"."T 3" (
c1 int NOT NULL,
c2 int NOT NULL,
c3 text,
CONSTRAINT t3_pkey PRIMARY KEY (c1)
);
CREATE TABLE "S 1"."T 4" (
c1 int NOT NULL,
c2 int NOT NULL,
c3 text,
CONSTRAINT t4_pkey PRIMARY KEY (c1)
);
-- Disable autovacuum for these tables to avoid unexpected effects of that
ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
ALTER TABLE "S 1"."T 3" SET (autovacuum_enabled = 'false');
ALTER TABLE "S 1"."T 4" SET (autovacuum_enabled = 'false');
\c contrib_regression postgres
SET client_min_messages TO notice;
\set VERBOSITY terse
\set QUIET off
-- ===================================================================
-- Test listing remote schemas
-- ===================================================================
\echo '## Test listing of remote schemas without permissions before the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
\c contrib_regression postgres
\echo '## Test listing of remote schemas (sunny day)'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
\echo '## Test listing of remote schemas without permissions after the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
\c contrib_regression postgres
\echo '## Test listing of remote schemas with permissions (sunny day)'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
\c contrib_regression postgres
\echo '## Test listing of remote schemas without permissions after revoking access (rainy day)'
SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback');
\c contrib_regression postgres
\echo '## Test listing of remote schemas (rainy day): Server does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'Does Not Exist');
-- ===================================================================
-- Test listing remote tables
-- ===================================================================
\echo '## Test listing of remote tables without permissions before the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
\c contrib_regression postgres
\echo '## Test listing of remote tables (sunny day)'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
\echo '## Test listing of remote tables without permissions after the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
\c contrib_regression postgres
\echo '## Test listing of remote tables with permissions (sunny day)'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
\c contrib_regression postgres
\echo '## Test listing of remote tables without permissions after revoking access (rainy day)'
SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'S 1');
\c contrib_regression postgres
\echo '## Test listing of remote tables (rainy day): Server does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'Does Not Exist', remote_schema => 'S 1');
\echo '## Test listing of remote tables (rainy day): Remote schema does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'Does Not Exist');
-- ===================================================================
-- Test listing remote columns
-- ===================================================================
\echo '## Test listing of remote columns without permissions before the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
\c contrib_regression postgres
\echo '## Test listing of remote columns (sunny day)'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
\echo '## Test listing of remote columns without permissions after the first instantiation (rainy day)'
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
\c contrib_regression postgres
\echo '## Test listing of remote columns with permissions (sunny day)'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
\c contrib_regression postgres
\echo '## Test listing of remote columns without permissions after revoking access (rainy day)'
SELECT cartodb.CDB_Federated_Server_Revoke_Access(server => 'loopback', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 1');
\c contrib_regression postgres
\echo '## Test listing of remote columns (rainy day): Server does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'Does Not Exist', remote_schema => 'S 1', remote_table => 'T 1');
\echo '## Test listing of remote columns (rainy day): Remote schema does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'Does Not Exist', remote_table => 'T 1');
\echo '## Test listing of remote columns (rainy day): Remote table does not exist'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'Does Not Exist');
\echo '## Test listing of remote columns (rainy day): Remote table is NULL'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => NULL::text);
-- ===================================================================
-- Test that using a different user to list tables and dropping it
-- does not break the server: We use loopback2 as it's in a clean state
-- ===================================================================
\echo '## Test listing of remote objects with permissions (sunny day)'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester2'::name);
\c contrib_regression cdb_fs_tester2
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2');
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1');
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1');
\c contrib_regression postgres
\echo '## Test that dropping the granted user works fine (sunny day)'
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2;
DROP ROLE cdb_fs_tester2;
\echo '## Test listing of remote objects with other user still works (sunny day)'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server => 'loopback2', db_role => 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback2');
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback2', remote_schema => 'S 1');
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback2', remote_schema => 'S 1', remote_table => 'T 1');
-- ===================================================================
-- Cleanup 1
-- ===================================================================
\set QUIET on
\c cdb_fs_tester cdb_fs_tester
DROP TABLE "S 1". "T 1";
DROP TABLE "S 1". "T 2";
DROP TABLE "S 1". "T 3";
DROP TABLE "S 1". "T 4";
DROP SCHEMA "S 1";
DROP TYPE user_enum;
-- ===================================================================
-- Setup 2: Using Postgis too
-- ===================================================================
\c cdb_fs_tester postgres
CREATE EXTENSION postgis;
\c cdb_fs_tester cdb_fs_tester
CREATE SCHEMA "S 1";
CREATE TABLE "S 1"."T 5" (
geom geometry(Geometry,4326),
geom_wm geometry(GeometryZ,3857),
geo_nosrid geometry,
geog geography
);
\c contrib_regression postgres
SET client_min_messages TO notice;
\set VERBOSITY terse
\set QUIET off
-- ===================================================================
-- Test the listing functions
-- ===================================================================
\echo '## Test listing of remote geometry columns (sunny day)'
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5');
\echo '## Test listing of remote geometry columns (sunny day) - Rerun'
-- Rerun should be ok
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Columns(server => 'loopback', remote_schema => 'S 1', remote_table => 'T 5');
-- ===================================================================
-- Test invalid password
-- ===================================================================
\echo '## Check error message with invalid password (rainy day)'
SELECT cartodb.CDB_Federated_Server_Register_PG(server => 'loopback_invalid'::text, config => '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "cdb_fs_tester",
"password": "wrong password"
}
}'::jsonb);
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Schemas(server => 'loopback_invalid');
SELECT cartodb.CDB_Federated_Server_Unregister(server => 'loopback_invalid'::text);
-- ===================================================================
-- Cleanup 2
-- ===================================================================
\set QUIET on
\c cdb_fs_tester cdb_fs_tester
DROP TABLE "S 1". "T 5";
DROP SCHEMA "S 1";
\c contrib_regression postgres
\set QUIET on
SET client_min_messages TO error;
\set VERBOSITY terse
SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server => 'loopback'::text);
SELECT 'D2', cartodb.CDB_Federated_Server_Unregister(server => 'loopback2'::text);
DROP DATABASE cdb_fs_tester;
-- Drop role
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
DROP ROLE cdb_fs_tester;
DROP EXTENSION postgres_fdw;
\set QUIET off

162
test/CDB_FederatedServerListRemote_expect Normal file
View File

@@ -0,0 +1,162 @@
C1|
C2|
## Test listing of remote schemas without permissions before the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote schemas (sunny day)
S 1
information_schema
public
## Test listing of remote schemas without permissions after the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote schemas with permissions (sunny day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
S 1
information_schema
public
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote schemas without permissions after revoking access (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote schemas (rainy day): Server does not exist
ERROR: Server "Does Not Exist" does not exist
## Test listing of remote tables without permissions before the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote tables (sunny day)
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback"
f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
## Test listing of remote tables without permissions after the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote tables with permissions (sunny day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback"
f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote tables without permissions after revoking access (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote tables (rainy day): Server does not exist
ERROR: Server "Does Not Exist" does not exist
## Test listing of remote tables (rainy day): Remote schema does not exist
## Test listing of remote columns without permissions before the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote columns (sunny day)
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback"
C 1|integer
c2|integer
c3|text
c4|timestamp with time zone
c5|timestamp without time zone
c6|character varying
c7|character
c8|USER-DEFINED
## Test listing of remote columns without permissions after the first instantiation (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote columns with permissions (sunny day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback"
C 1|integer
c2|integer
c3|text
c4|timestamp with time zone
c5|timestamp without time zone
c6|character varying
c7|character
c8|USER-DEFINED
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote columns without permissions after revoking access (rainy day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
You are now connected to database "contrib_regression" as user "postgres".
## Test listing of remote columns (rainy day): Server does not exist
ERROR: Server "Does Not Exist" does not exist
## Test listing of remote columns (rainy day): Remote schema does not exist
## Test listing of remote columns (rainy day): Remote table does not exist
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback"
## Test listing of remote columns (rainy day): Remote table is NULL
ERROR: Remote table name cannot be NULL
## Test listing of remote objects with permissions (sunny day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
S 1
information_schema
public
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2"
f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2"
C 1|integer
c2|integer
c3|text
c4|timestamp with time zone
c5|timestamp without time zone
c6|character varying
c7|character
c8|USER-DEFINED
You are now connected to database "contrib_regression" as user "postgres".
## Test that dropping the granted user works fine (sunny day)
REVOKE
DROP ROLE
## Test listing of remote objects with other user still works (sunny day)
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
S 1
information_schema
public
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2"
f|T 1|||||[{"Name" : "C 1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}, {"Name" : "c4", "Type" : "timestamp with time zone"}, {"Name" : "c5", "Type" : "timestamp without time zone"}, {"Name" : "c6", "Type" : "character varying"}, {"Name" : "c7", "Type" : "character"}, {"Name" : "c8", "Type" : "USER-DEFINED"}]
f|T 2|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "text"}]
f|T 3|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
f|T 4|||||[{"Name" : "c1", "Type" : "integer"}, {"Name" : "c2", "Type" : "integer"}, {"Name" : "c3", "Type" : "text"}]
INFO: Could not find Postgis installation in the remote "public" schema in server "loopback2"
C 1|integer
c2|integer
c3|text
c4|timestamp with time zone
c5|timestamp without time zone
c6|character varying
c7|character
c8|USER-DEFINED
## Test listing of remote geometry columns (sunny day)
geo_nosrid|GEOMETRY,0
geog|Geometry,0
geom|GEOMETRY,4326
geom_wm|GEOMETRY,3857
## Test listing of remote geometry columns (sunny day) - Rerun
geo_nosrid|GEOMETRY,0
geog|Geometry,0
geom|GEOMETRY,4326
geom_wm|GEOMETRY,3857
## Check error message with invalid password (rainy day)
ERROR: could not connect to server "cdb_fs_loopback_invalid"
D1|
D2|

405
test/CDB_FederatedServerTables.sql Normal file
View File

@@ -0,0 +1,405 @@
-- ===================================================================
-- create FDW objects
-- ===================================================================
\set QUIET on
SET client_min_messages TO error;
\set VERBOSITY terse
CREATE EXTENSION postgres_fdw;
-- We create a username following the same steps as organization members
CREATE ROLE cdb_fs_tester LOGIN PASSWORD 'cdb_fs_passwd';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester;
CREATE ROLE cdb_fs_tester2 LOGIN PASSWORD 'cdb_fs_passwd2';
GRANT CONNECT ON DATABASE contrib_regression TO cdb_fs_tester2;
-- Create database to be used as remote
CREATE DATABASE cdb_fs_tester OWNER cdb_fs_tester;
SELECT 'C1', cartodb.CDB_Federated_Server_Register_PG(server := 'loopback'::text, config := '{
"server": {
"host": "localhost",
"port": @@PGPORT@@
},
"credentials": {
"username": "cdb_fs_tester",
"password": "cdb_fs_passwd"
}
}'::jsonb);
-- ===================================================================
-- create objects used through FDW loopback server
-- ===================================================================
\c cdb_fs_tester postgres
CREATE EXTENSION postgis;
\c cdb_fs_tester cdb_fs_tester
CREATE SCHEMA remote_schema;
CREATE TABLE remote_schema.remote_geom(id int, another_field text, geom geometry(Geometry,4326));
INSERT INTO remote_schema.remote_geom VALUES (1, 'patata', 'SRID=4326;POINT(1 1)'::geometry);
INSERT INTO remote_schema.remote_geom VALUES (2, 'patata2', 'SRID=4326;POINT(2 2)'::geometry);
CREATE TABLE remote_schema.remote_geom2(cartodb_id bigint, another_field text, the_geom geometry(Geometry,4326), the_geom_webmercator geometry(Geometry,3857));
INSERT INTO remote_schema.remote_geom2 VALUES (3, 'patata', 'SRID=4326;POINT(3 3)'::geometry, 'SRID=3857;POINT(3 3)');
CREATE TABLE remote_schema.remote_other(id bigint, field text, field2 text);
INSERT INTO remote_schema.remote_other VALUES (1, 'delicious', 'potatoes');
CREATE TABLE remote_schema.remote_geom3(id bigint, geom geometry(Geometry,4326), geom_mercator geometry(Geometry,3857));
-- ===================================================================
-- Test the listing functions
-- ===================================================================
\c contrib_regression postgres
SET client_min_messages TO error;
\set VERBOSITY terse
\set QUIET off
\echo '## Registering an existing table works'
SELECT 'R1', cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom'
);
SELECT 'S1', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback.remote_geom;
Select * FROM CDB_Federated_Server_List_Remote_Tables(
server => 'loopback',
remote_schema => 'remote_schema'
);
\echo '## Registering another existing table works'
SELECT 'R2', cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom2',
id_column => 'cartodb_id',
geom_column => 'the_geom',
webmercator_column => 'the_geom_webmercator',
local_name => 'myFullTable'
);
SELECT 'S2', cartodb_id, ST_AsText(the_geom), another_field FROM cdb_fs_loopback."myFullTable";
Select * FROM CDB_Federated_Server_List_Remote_Tables(
server => 'loopback',
remote_schema => 'remote_schema'
);
\echo '## Re-registering a table works'
SELECT 'R3', cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom2',
id_column => 'cartodb_id',
-- Switch geom and geom_column on purpose to force ST_Transform to be used
geom_column => 'the_geom_webmercator',
webmercator_column => 'the_geom',
local_name => 'different_name'
);
-- The old view should dissapear
SELECT 'S3_old', cartodb_id, another_field FROM cdb_fs_loopback."myFullTable";
-- And the new appear
SELECT 'S3_new', cartodb_id, another_field FROM cdb_fs_loopback.different_name;
\echo '## Unregistering works'
-- Deregistering the first table
SELECT 'U1', cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom'
);
-- Selecting from the created view should fail now
SELECT 'UCheck1', cartodb_id, ST_AsText(the_geom), another_field FROM remote_geom;
Select * FROM CDB_Federated_Server_List_Remote_Tables(
server => 'loopback',
remote_schema => 'remote_schema'
);
-- ===================================================================
-- Test input
-- ===================================================================
\echo '## Registering a table: Invalid server fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'Does not exist',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: NULL server fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => NULL::text,
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: Invalid schema fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'Does not exist',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: NULL schema fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => NULL::text,
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: Invalid table fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'Does not exist',
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: NULL table fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => NULL::text,
id_column => 'id',
geom_column => 'geom'
);
\echo '## Registering a table: Invalid id fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'Does not exist',
geom_column => 'geom'
);
\echo '## Registering a table: NULL id fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => NULL::text,
geom_column => 'geom'
);
\echo '## Registering a table: Invalid geom_column fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'Does not exists'
);
\echo '## Registering a table: NULL geom_column is OK: Reuses geom_mercator'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => NULL::text,
webmercator_column => 'geom'
);
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom';
SELECT cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom'
);
\echo '## Registering a table: Invalid webmercator_column fails'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
webmercator_column => 'Does not exists'
);
\echo '## Registering a table: NULL webmercator_column is OK: Reuses geom'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
webmercator_column => NULL::text
);
SELECT * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom';
SELECT cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom'
);
\echo '##Registering a table with extra columns show the correct information'
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom3',
id_column => 'id',
geom_column => 'geom',
webmercator_column => 'geom_mercator'
);
Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema') where remote_table = 'remote_geom3';
SELECT cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom3'
);
-- ===================================================================
-- Test conflicts
-- ===================================================================
\echo '## Target conflict is handled nicely: Table'
CREATE TABLE cdb_fs_loopback.localtable (a integer);
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
local_name => 'localtable');
\echo '## Target conflict is handled nicely: View'
CREATE VIEW cdb_fs_loopback.localtable2 AS Select * from cdb_fs_loopback.localtable;
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
local_name => 'localtable2');
DROP VIEW cdb_fs_loopback.localtable2;
DROP TABLE cdb_fs_loopback.localtable;
-- ===================================================================
-- Test permissions
-- ===================================================================
\echo '## Registering tables does not work without permissions'
\c contrib_regression cdb_fs_tester
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
local_name => 'localtable');
\echo '## Normal users can not write in the import schema'
CREATE TABLE cdb_fs_loopback.localtable (a integer);
\echo '## Listing remote tables does not work without permissions'
Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
\echo '## Registering tables works with granted permissions'
\c contrib_regression postgres
SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name);
\c contrib_regression cdb_fs_tester
SELECT cartodb.CDB_Federated_Table_Register(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom',
id_column => 'id',
geom_column => 'geom',
local_name => 'localtable');
\echo '## Listing remote tables works with granted permissions'
Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
\echo '## Selecting from a registered table with granted permissions works'
Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable;
\echo '## Selecting from a registered table without permissions does not work'
\c contrib_regression cdb_fs_tester2
CREATE OR REPLACE FUNCTION catch_permission_error(query text)
RETURNS bool
AS $$
BEGIN
EXECUTE query;
RETURN FALSE;
EXCEPTION
WHEN insufficient_privilege THEN
RETURN TRUE;
WHEN OTHERS THEN
RAISE WARNING 'Exception %', sqlstate;
RETURN FALSE;
END
$$ LANGUAGE 'plpgsql';
Select catch_permission_error($$SELECT cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable$$);
DROP FUNCTION catch_permission_error(text);
\echo '## Deleting a registered table without permissions does not work'
SELECT cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom'
);
\echo '## Only the owner can grant permissions over the server'
SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name);
\echo '## Everything works for a different user when granted permissions'
\c contrib_regression postgres
SELECT cartodb.CDB_Federated_Server_Grant_Access(server := 'loopback', db_role := 'cdb_fs_tester2'::name);
\c contrib_regression cdb_fs_tester2
Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
Select cartodb_id, ST_AsText(the_geom) from cdb_fs_loopback.localtable;
\echo '## A different user can unregister a table'
SELECT cartodb.CDB_Federated_Table_Unregister(
server => 'loopback',
remote_schema => 'remote_schema',
remote_table => 'remote_geom'
);
Select * FROM cartodb.CDB_Federated_Server_List_Remote_Tables(server => 'loopback', remote_schema => 'remote_schema');
\echo '## Only the owner can revoke permissions over the server'
SELECT cartodb.CDB_Federated_Server_Revoke_Access(server := 'loopback', db_role := 'cdb_fs_tester'::name);
-- ===================================================================
-- Cleanup
-- ===================================================================
\set QUIET on
\c contrib_regression postgres
SET client_min_messages TO error;
\set VERBOSITY terse
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester2;
DROP ROLE cdb_fs_tester2;
SELECT 'D1', cartodb.CDB_Federated_Server_Unregister(server := 'loopback'::text);
DROP DATABASE cdb_fs_tester;
REVOKE CONNECT ON DATABASE contrib_regression FROM cdb_fs_tester;
DROP ROLE cdb_fs_tester;
DROP EXTENSION postgres_fdw;
\set QUIET off

116
test/CDB_FederatedServerTables_expect Normal file
View File

@@ -0,0 +1,116 @@
C1|
## Registering an existing table works
R1|
S1|1|POINT(1 1)|patata
S1|2|POINT(2 2)|patata2
t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
f|remote_geom2|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
## Registering another existing table works
R2|
S2|3|POINT(3 3)|patata
t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
t|remote_geom2|cdb_fs_loopback."myFullTable"|cartodb_id|the_geom|the_geom_webmercator|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
## Re-registering a table works
R3|
ERROR: relation "cdb_fs_loopback.myFullTable" does not exist at character 49
S3_new|3|patata
## Unregistering works
U1|
ERROR: relation "remote_geom" does not exist at character 71
f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
## Registering a table: Invalid server fails
ERROR: Server "Does not exist" does not exist
## Registering a table: NULL server fails
ERROR: Server name cannot be NULL
## Registering a table: Invalid schema fails
ERROR: Could not import schema "Does not exist" of server "loopback": schema "Does not exist" is not present on foreign server "cdb_fs_loopback"
## Registering a table: NULL schema fails
ERROR: Schema name cannot be NULL
## Registering a table: Invalid table fails
ERROR: Could not import table "remote_schema.Does not exist" of server "loopback"
## Registering a table: NULL table fails
ERROR: Remote table name cannot be NULL
## Registering a table: Invalid id fails
ERROR: non integer id_column "Does not exist"
## Registering a table: NULL id fails
ERROR: non integer id_column "<NULL>"
## Registering a table: Invalid geom_column fails
ERROR: non geometry column "Does not exists"
## Registering a table: NULL geom_column is OK: Reuses geom_mercator
t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
## Registering a table: Invalid webmercator_column fails
ERROR: non geometry column "Does not exists"
## Registering a table: NULL webmercator_column is OK: Reuses geom
t|remote_geom|cdb_fs_loopback.remote_geom|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
##Registering a table with extra columns show the correct information
t|remote_geom3|cdb_fs_loopback.remote_geom3|id|geom|geom_mercator|[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
## Target conflict is handled nicely: Table
CREATE TABLE
ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable" already exists
## Target conflict is handled nicely: View
CREATE VIEW
ERROR: Could not import table "remote_geom" as "cdb_fs_loopback.localtable2" already exists
DROP VIEW
DROP TABLE
## Registering tables does not work without permissions
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to access the server "loopback"
## Normal users can not write in the import schema
ERROR: permission denied for schema cdb_fs_loopback at character 14
## Listing remote tables does not work without permissions
ERROR: Not enough permissions to access the server "loopback"
## Registering tables works with granted permissions
You are now connected to database "contrib_regression" as user "postgres".
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
## Listing remote tables works with granted permissions
t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
## Selecting from a registered table with granted permissions works
1|POINT(1 1)
2|POINT(2 2)
## Selecting from a registered table without permissions does not work
You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
CREATE FUNCTION
t
DROP FUNCTION
## Deleting a registered table without permissions does not work
ERROR: Not enough permissions to access the server "loopback"
## Only the owner can grant permissions over the server
ERROR: You do not have rights to grant access on "loopback"
## Everything works for a different user when granted permissions
You are now connected to database "contrib_regression" as user "postgres".
You are now connected to database "contrib_regression" as user "cdb_fs_tester2".
t|remote_geom|cdb_fs_loopback.localtable|id|geom|geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
1|POINT(1 1)
2|POINT(2 2)
## A different user can unregister a table
NOTICE: drop cascades to view cdb_fs_loopback.localtable
f|remote_geom|||||[{"Name" : "another_field", "Type" : "text"}, {"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "id", "Type" : "integer"}]
t|remote_geom2|cdb_fs_loopback.different_name|cartodb_id|the_geom_webmercator|the_geom|[{"Name" : "another_field", "Type" : "text"}, {"Name" : "cartodb_id", "Type" : "bigint"}, {"Name" : "the_geom", "Type" : "GEOMETRY,4326"}, {"Name" : "the_geom_webmercator", "Type" : "GEOMETRY,3857"}]
f|remote_geom3|||||[{"Name" : "geom", "Type" : "GEOMETRY,4326"}, {"Name" : "geom_mercator", "Type" : "GEOMETRY,3857"}, {"Name" : "id", "Type" : "bigint"}]
f|remote_other|||||[{"Name" : "field", "Type" : "text"}, {"Name" : "field2", "Type" : "text"}, {"Name" : "id", "Type" : "bigint"}]
## Only the owner can revoke permissions over the server
ERROR: You do not have rights to revoke access on "loopback"
D1|

69
test/CDB_FederatedServer_expect Normal file
View File

@@ -0,0 +1,69 @@
## List empty servers shows nothing
## List non-existent server shows nothing
## Create and list a server works
1.3|
1.4|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
## Create and list a second server works
2.1|
2.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
2.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,fdw_user)
## List server by name works
2.3|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
## Re-register a second server works
3.1|
3.2|(myRemote,postgres_fdw,localhost,5432,,read-only,fdw_user)
3.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user)
## Unregister server 1 works
4.1|
4.2|(myRemote2,postgres_fdw,localhost,5432,fdw_target,read-only,other_remote_user)
## Unregistering a server that does not exist fails
ERROR: Server "doesNotExist" does not exist
## Unregister the second server works
6.1|
## Create a server with NULL name fails
ERROR: Server name cannot be NULL
## Create a server with NULL config fails
7.01|
## Create a server with empty config fails
ERROR: Server information is mandatory
## Create a server without credentials fails
ERROR: Credentials are mandatory
## Create a server with empty credentials works
7.3|
7.4|(empty,postgres_fdw,localhost,5432,fdw_target,read-only,)
7.5|
## Create a server without options fails
ERROR: Server information is mandatory
## Create a server with special characters works
8.1|
8.2|("myRemote"" or'not",postgres_fdw,localhost,5432,"fdw target",read-only,"fdw user")
8.3|
9.1|
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
## All users are able to list servers
9.2|(myRemote3,postgres_fdw,localhost,5432,,read-only,)
## Only superadmins can create servers
ERROR: Could not create server myRemote4: permission denied for foreign-data wrapper postgres_fdw
You are now connected to database "contrib_regression" as user "postgres".
## Granting access to a user works
9.5|
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
9.55|(myRemote3,postgres_fdw,localhost,5432,,read-only,fdw_user)
You are now connected to database "contrib_regression" as user "postgres".
ERROR: Server "does not exist" does not exist
ERROR: Could not grant access on "myRemote3" to "does not exist": role "does not exist" does not exist
## Granting access again raises a notice
NOTICE: role "cdb_fs_tester" is already a member of role "cdb_fs_role_95b63382aabca4433e7bd9cba6c30368"
9.8|
## Revoking access to a user works
9.9|
9.10|
## Unregistering a server with active grants works
9.11|
## A user with granted access can not drop a server
10.1|
10.2|
You are now connected to database "contrib_regression" as user "cdb_fs_tester".
ERROR: Not enough permissions to drop the server "myRemote4"
You are now connected to database "contrib_regression" as user "postgres".
10.4|

2
test/CDB_SyncTableTest.sql
View File

@@ -51,7 +51,7 @@ UPDATE test_sync_dest SET the_geom = cartodb.CDB_LatLng(lat, lon); -- A "gecodin
SET client_min_messages TO notice;
SELECT cartodb.CDB_SyncTable('test_sync_source', 'public', 'test_sync_dest', '{the_geom, the_geom_webmercator}');
SELECT * FROM test_sync_source ORDER BY cartodb_id;
SELECT * FROM test_sync_dest ORDER BY cartodb_id;
SELECT cartodb_id, the_geom, lat, lon, name FROM test_sync_dest ORDER BY cartodb_id;
\echo 'It will work with schemas that need quoting'
\set QUIET on

8
test/CDB_SyncTableTest_expect
View File

@@ -54,10 +54,10 @@ NOTICE: MODIFIED 0 row(s)
2|||2|2|bar
4|||4|4|cantaloupe
5|||5|5|sandia
1|0101000020E6100000000000000000F03F000000000000F03F|0101000020110F0000DB0B4ADA772DFB402B432E49D22DFB40|1|1|foo
2|0101000020E610000000000000000000400000000000000040|0101000020110F00003C0C4ADA772D0B4177F404ABE12E0B41|2|2|bar
4|0101000020E610000000000000000010400000000000001040|0101000020110F00003C0C4ADA772D1B4160AB497020331B41|4|4|cantaloupe
5|0101000020E610000000000000000014400000000000001440|0101000020110F000099476EE86AFC20413E7EB983F2012141|5|5|sandia
1|0101000020E6100000000000000000F03F000000000000F03F|1|1|foo
2|0101000020E610000000000000000000400000000000000040|2|2|bar
4|0101000020E610000000000000000010400000000000001040|4|4|cantaloupe
5|0101000020E610000000000000000014400000000000001440|5|5|sandia
It will work with schemas that need quoting
INSERT 0 1

2
test/CDB_Username_expect
View File

@@ -1,4 +1,4 @@
postgres
@@PGUSER@@
fulano
fulanito

62
test/extension/test.sh
View File

@@ -598,68 +598,6 @@ test_extension|public|"local-table-with-dashes"'
sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;"
sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't'
# Check user-defined FDW's
# Set up a user foreign server
read -d '' ufdw_config <<- EOF
{
"server": {
"extensions": "postgis",
"dbname": "fdw_target",
"host": "localhost",
"port": ${PGPORT:-5432}
},
"user_mapping": {
"user": "fdw_user",
"password": "foobarino"
}
}
EOF
sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('user-defined-test', '$ufdw_config');"
# Grant a user access to that FDW, and to grant to others
sql postgres 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_1 WITH ADMIN OPTION;'
# Set up a user foreign table
sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('user-defined-test', 'test_fdw', 'foo');"
# Check that the table can be accessed by the owner/creator
sql cdb_testmember_1 'SELECT * from "cdb_fdw_user-defined-test".foo;'
sql cdb_testmember_1 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
# Check that a role with no permissions cannot use the FDW to access a remote table
sql cdb_testmember_2 'IMPORT FOREIGN SCHEMA test_fdw LIMIT TO (foo) FROM SERVER "cdb_fdw_user-defined-test" INTO public' fails
# Check that the table can be accessed by some other user by granting the role
sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO cdb_testmember_2;'
sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM cdb_testmember_2;'
# Check that the table can be accessed by org members
sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_user-defined-test');"
sql cdb_testmember_2 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
sql cdb_testmember_1 "SELECT cartodb.CDB_Organization_Revoke_Role('cdb_fdw_user-defined-test');"
# By default publicuser cannot access the FDW
sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' fails
sql cdb_testmember_1 'GRANT "cdb_fdw_user-defined-test" TO publicuser;' # but can be granted
sql publicuser 'SELECT a from "cdb_fdw_user-defined-test".foo LIMIT 1;' should 42
sql cdb_testmember_1 'REVOKE "cdb_fdw_user-defined-test" FROM publicuser;'
# If there are dependent objects, we cannot drop the foreign server
sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')" fails
sql cdb_testmember_1 'DROP FOREIGN TABLE "cdb_fdw_user-defined-test".foo;'
sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('user-defined-test')"
# But if there are, we can set the force flag to true to drop everything (defaults to false)
sql postgres "SELECT cartodb._CDB_SetUp_User_PG_FDW_Server('another_user_defined_test', '$ufdw_config');"
sql postgres 'GRANT cdb_fdw_another_user_defined_test TO cdb_testmember_1 WITH ADMIN OPTION;'
sql cdb_testmember_1 "SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('another_user_defined_test', 'test_fdw', 'foo');"
sql postgres "SELECT cartodb._CDB_Drop_User_PG_FDW_Server('another_user_defined_test', /* force = */ true)"
# Teardown
DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'
DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'