cartodb4/cartodb-postgresql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Revert 0.33.0"

Browse Source 
This reverts commit 24cb6cf9c1.
This commit is contained in:
Raúl Marín
2019-12-24 00:33:35 +01:00
parent 8974986d61
commit 9010642326
23 changed files with 2793 additions and 307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

452
scripts-available/CDB_FederatedServer.sql
View File

@@ -1,15 +1,437 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Name_Pattern();
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Name(TEXT, BOOL);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Extract_Server_Name(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Schema_Name(NAME, TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Generate_Server_Role_Name(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Create_Schema(NAME, TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_server_type(NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_credentials_to_user_mapping(JSONB);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_add_default_options(jsonb);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_get_usermapping_username(NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Register_PG(TEXT, JSONB);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Unregister(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Servers(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Grant_Access(TEXT, NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Revoke_Access(TEXT, NAME);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- This function is just a placement to store and use the pattern for
-- foreign object names
-- Servers: cdb_fs_$(server_name)
-- View schema: cdb_fs_$(server_name)
-- > This is where all views created when importing tables are placed
-- > One server has only one view schema
-- Import Schemas: cdb_fs_schema_$(md5sum(server_name || remote_schema_name))
-- > This is where the foreign tables are placed
-- > One server has one import schema per remote schema plus auxiliar ones used
-- to access the remote catalog (pg_catalog, information_schema...)
-- Owner role: cdb_fs_$(md5sum(current_database() || server_name)
-- > This is the role than owns all schemas and tables related to the server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Name_Pattern()
RETURNS TEXT
AS $$
SELECT 'cdb_fs_'::text;
$$
LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid DB name for servers generated for the Federated Server
-- If check_existence is true, it'll throw if the server doesn't exists
-- This name is also used as the schema to store views
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Name(input_name TEXT, check_existence BOOL)
RETURNS NAME
AS $$
DECLARE
internal_server_name text := format('%s%s', @extschema@.__CDB_FS_Name_Pattern(), input_name);
BEGIN
IF input_name IS NULL OR char_length(input_name) = 0 THEN
RAISE EXCEPTION 'Server name cannot be NULL';
END IF;
-- We discard anything that would be truncated
IF (char_length(internal_server_name) >= 64) THEN
RAISE EXCEPTION 'Server name (%) is too long to be used as identifier', input_name;
END IF;
IF (check_existence AND (NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = internal_server_name))) THEN
RAISE EXCEPTION 'Server "%" does not exist', input_name;
END IF;
RETURN internal_server_name::name;
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
--
-- Given the internal name for a remote server, it returns the name used by the user
-- Reverses __CDB_FS_Generate_Server_Name
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Extract_Server_Name(internal_server_name NAME)
RETURNS TEXT
AS $$
SELECT right(internal_server_name,
char_length(internal_server_name::TEXT) - char_length(@extschema@.__CDB_FS_Name_Pattern()))::TEXT;
$$
LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid name for a schema generated for the Federated Server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name NAME, schema_name TEXT)
RETURNS NAME
AS $$
DECLARE
hash_value text := md5(internal_server_name::text || '__' || schema_name::text);
BEGIN
IF schema_name IS NULL THEN
RAISE EXCEPTION 'Schema name cannot be NULL';
END IF;
RETURN format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'schema_', hash_value)::name;
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
--
-- Produce a valid name for a role generated for the Federated Server
-- This needs to include the current database in its hash to avoid collisions in clusters with more than one database
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name NAME)
RETURNS NAME
AS $$
DECLARE
hash_value text := md5(current_database()::text || '__' || internal_server_name::text);
role_name text := format('%s%s%s', @extschema@.__CDB_FS_Name_Pattern(), 'role_', hash_value);
BEGIN
RETURN role_name::name;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL SAFE;
--
-- Creates (if not exist) a schema to place the objects for a remote schema
-- The schema is with the same AUTHORIZATION as the server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Create_Schema(internal_server_name NAME, schema_name TEXT)
RETURNS NAME
AS $$
DECLARE
schema_name name := @extschema@.__CDB_FS_Generate_Schema_Name(internal_server_name, schema_name);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
BEGIN
-- By changing the local role to the owner of the server we have an
-- easy way to check for permissions and keep all objects under the same owner
BEGIN
EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
EXCEPTION
WHEN invalid_parameter_value THEN
RAISE EXCEPTION 'Server "%" does not exist',
@extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(internal_server_name);
END;
IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = schema_name) THEN
EXECUTE 'CREATE SCHEMA ' || quote_ident(schema_name) || ' AUTHORIZATION ' || quote_ident(role_name);
END IF;
RETURN schema_name;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Returns the type of a server by internal name
-- Currently all of them should be postgres_fdw
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_server_type(internal_server_name NAME)
RETURNS name
AS $$
SELECT f.fdwname
FROM pg_foreign_server s
JOIN pg_foreign_data_wrapper f ON s.srvfdw = f.oid
WHERE s.srvname = internal_server_name;
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Take a config jsonb and transform it to an input suitable for _CDB_SetUp_User_PG_FDW_Server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_credentials_to_user_mapping(input_config JSONB)
RETURNS jsonb
AS $$
DECLARE
mapping jsonb := '{}'::jsonb;
BEGIN
IF NOT (input_config ? 'credentials') THEN
RAISE EXCEPTION 'Credentials are mandatory';
END IF;
-- For now, allow not passing username or password
IF input_config->'credentials'->'username' IS NOT NULL THEN
mapping := jsonb_build_object('user', input_config->'credentials'->'username');
END IF;
IF input_config->'credentials'->'password' IS NOT NULL THEN
mapping := mapping || jsonb_build_object('password', input_config->'credentials'->'password');
END IF;
RETURN (input_config - 'credentials')::jsonb || jsonb_build_object('user_mapping', mapping);
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-- Take a config jsonb as input and return it augmented with default
-- options
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_add_default_options(input_config jsonb)
RETURNS jsonb
AS $$
DECLARE
default_options jsonb := '{
"extensions": "postgis",
"updatable": "false",
"use_remote_estimate": "true",
"fetch_size": "1000"
}';
server_config jsonb;
BEGIN
IF NOT (input_config ? 'server') THEN
RAISE EXCEPTION 'Server information is mandatory';
END IF;
server_config := default_options || to_jsonb(input_config->'server');
RETURN jsonb_set(input_config, '{server}'::text[], server_config);
END
$$
LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE;
-- Given an server name, returns the username used in the configuration if the caller has rights to access it
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_get_usermapping_username(internal_server_name NAME)
RETURNS text
AS $$
DECLARE
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(internal_server_name);
username text;
BEGIN
BEGIN
EXECUTE 'SET LOCAL ROLE ' || quote_ident(role_name);
EXCEPTION WHEN OTHERS THEN
RETURN NULL;
END;
SELECT (SELECT option_value FROM pg_options_to_table(u.umoptions) WHERE option_name LIKE 'user') as name INTO username
FROM pg_foreign_server s
LEFT JOIN pg_user_mappings u
ON u.srvid = s.oid
WHERE s.srvname = internal_server_name
ORDER BY 1;
RESET ROLE;
RETURN username;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Registers a new PG server
--
-- Example config: '{
-- "server": {
-- "dbname": "fdw_target",
-- "host": "localhost",
-- "port": 5432,
-- "extensions": "postgis",
-- "updatable": "false",
-- "use_remote_estimate": "true",
-- "fetch_size": "1000"
-- },
-- "credentials": {
-- "username": "fdw_user",
-- "password": "foobarino"
-- }
-- }'
--
-- The configuration from __CDB_FS_add_default_options will be appended
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Register_PG(server TEXT, config JSONB)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
final_config json := @extschema@.__CDB_FS_credentials_to_user_mapping(@extschema@.__CDB_FS_add_default_options(config));
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
row record;
option record;
BEGIN
IF NOT EXISTS (SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
RAISE EXCEPTION 'postgres_fdw extension is not installed'
USING HINT = 'Please install it with `CREATE EXTENSION postgres_fdw`';
END IF;
-- We only create server and roles if the server didn't exist before
IF NOT EXISTS (SELECT * FROM pg_foreign_server WHERE srvname = server_internal) THEN
BEGIN
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', server_internal);
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_name) THEN
EXECUTE FORMAT('CREATE ROLE %I NOLOGIN', role_name);
END IF;
EXECUTE FORMAT('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', current_database(), role_name);
-- These grants over `@extschema@` and `@postgisschema@` are necessary for the cases
-- where the schemas aren't accessible to PUBLIC, which is what happens in a CARTO database
EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@extschema@', role_name);
EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@extschema@', role_name);
EXECUTE FORMAT('GRANT USAGE ON SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT SELECT ON ALL TABLES IN SCHEMA %I TO %I', '@postgisschema@', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', role_name);
EXECUTE FORMAT('GRANT USAGE ON FOREIGN SERVER %I TO %I', server_internal, role_name);
EXECUTE FORMAT('ALTER SERVER %I OWNER TO %I', server_internal, role_name);
EXECUTE FORMAT ('CREATE USER MAPPING FOR %I SERVER %I', role_name, server_internal);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not create server %: %', server, SQLERRM
USING HINT = 'Please clean the left over objects';
END;
END IF;
-- Add new options
FOR row IN SELECT p.key, p.value from lateral json_each_text(final_config->'server') p
LOOP
IF NOT EXISTS (
WITH a AS (
SELECT split_part(unnest(srvoptions), '=', 1) AS options FROM pg_foreign_server WHERE srvname=server_internal
) SELECT * from a where options = row.key)
THEN
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', server_internal, row.key, row.value);
ELSE
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', server_internal, row.key, row.value);
END IF;
END LOOP;
-- Update user mapping settings
FOR option IN SELECT o.key, o.value from lateral json_each_text(final_config->'user_mapping') o
LOOP
IF NOT EXISTS (
WITH a AS (
SELECT split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = server_internal AND usename = role_name
) SELECT * from a where options = option.key)
THEN
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (ADD %I %L)', role_name, server_internal, option.key, option.value);
ELSE
EXECUTE FORMAT('ALTER USER MAPPING FOR %I SERVER %I OPTIONS (SET %I %L)', role_name, server_internal, option.key, option.value);
END IF;
END LOOP;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Drops a registered server and all the objects associated with it
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Unregister(server TEXT)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
SET client_min_messages = ERROR;
BEGIN
EXECUTE FORMAT ('DROP USER MAPPING FOR %I SERVER %I', role_name, server_internal);
EXECUTE FORMAT ('DROP OWNED BY %I', role_name);
EXECUTE FORMAT ('DROP ROLE %I', role_name);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to drop the server "%"', server;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List registered servers
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Servers(server TEXT DEFAULT '%')
RETURNS TABLE (
name text,
driver text,
host text,
port text,
dbname text,
readmode text,
username text
)
AS $$
DECLARE
server_name text := concat(@extschema@.__CDB_FS_Name_Pattern(), server);
BEGIN
RETURN QUERY SELECT
-- Name as shown to the user
@extschema@.__CDB_FS_Extract_Server_Name(s.srvname) AS "Name",
-- Which driver are we using (postgres_fdw, odbc_fdw...)
@extschema@.__CDB_FS_server_type(s.srvname)::text AS "Driver",
-- Read options from pg_foreign_server
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'host') AS "Host",
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'port') AS "Port",
(SELECT option_value FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'dbname') AS "DBName",
CASE WHEN (SELECT NOT option_value::boolean FROM pg_options_to_table(s.srvoptions) WHERE option_name LIKE 'updatable') THEN 'read-only' ELSE 'read-write' END AS "ReadMode",
@extschema@.__CDB_FS_get_usermapping_username(s.srvname)::text AS "Username"
FROM pg_foreign_server s
LEFT JOIN pg_user_mappings u
ON u.srvid = s.oid
WHERE s.srvname ILIKE server_name
ORDER BY 1;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL SAFE;
--
-- Grant access to a server
-- In the future we might consider adding the server's view schema to the role search_path
-- to make it easier to access the created views
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Grant_Access(server TEXT, db_role NAME)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
IF (db_role IS NULL) THEN
RAISE EXCEPTION 'User role "%" cannot be NULL', username;
END IF;
BEGIN
EXECUTE format('GRANT %I TO %I', server_role_name, db_role);
EXCEPTION
WHEN insufficient_privilege THEN
RAISE EXCEPTION 'You do not have rights to grant access on "%"', server;
WHEN OTHERS THEN
RAISE EXCEPTION 'Could not grant access on "%" to "%": %', server, db_role, SQLERRM;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Revoke access to a server
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Revoke_Access(server TEXT, db_role NAME)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
BEGIN
IF (db_role IS NULL) THEN
RAISE EXCEPTION 'User role "%" cannot be NULL', username;
END IF;
BEGIN
EXECUTE format('REVOKE %I FROM %I', server_role_name, db_role);
EXCEPTION
WHEN insufficient_privilege THEN
RAISE EXCEPTION 'You do not have rights to revoke access on "%"', server;
WHEN OTHERS THEN
RAISE EXCEPTION 'Could not revoke access on "%" to "%": %', server, db_role, SQLERRM;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

252
scripts-available/CDB_FederatedServerDiagnostics.sql
View File

@@ -1,9 +1,243 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Import_If_Not_Exists(name, name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Version_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Options_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Host_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Foreign_Server_Port_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(name, float, integer);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Server_Diagnostics_PG(name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_Diagnostics(TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- Import a foreign table if it does not exist
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal name, remote_schema name, remote_table name)
RETURNS void
AS $$
DECLARE
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
remote_schema, remote_table, server_internal, local_schema);
END IF;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the version of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal name)
RETURNS text
AS $$
DECLARE
remote_schema name := 'pg_catalog';
remote_table name := 'pg_settings';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
remote_server_version text;
BEGIN
PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
BEGIN
EXECUTE format('
SELECT setting FROM %I.%I WHERE name = ''server_version'';
', local_schema, remote_table) INTO remote_server_version;
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
RETURN remote_server_version;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the PostGIS extension version of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal name)
RETURNS text
AS $$
DECLARE
remote_schema name := 'pg_catalog';
remote_table name := 'pg_extension';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
remote_postgis_version text;
BEGIN
PERFORM @extschema@.__CDB_FS_Import_If_Not_Exists(server_internal, remote_schema, remote_table);
BEGIN
EXECUTE format('
SELECT extversion FROM %I.%I WHERE extname = ''postgis'';
', local_schema, remote_table) INTO remote_postgis_version;
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
RETURN remote_postgis_version;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Get the foreign server options of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal name)
RETURNS jsonb
AS $$
-- See https://www.postgresql.org/docs/current/catalog-pg-foreign-server.html
-- See https://www.postgresql.org/docs/current/functions-info.html
SELECT jsonb_object_agg(opt.option_name, opt.option_value) FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server
WHERE srvname = server_internal
) AS opt;
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get a foreign PG server hostname from the catalog
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Host_PG(server_internal name)
RETURNS text
AS $$
SELECT option_value FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server WHERE srvname = server_internal
) AS opt WHERE opt.option_name = 'host';
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get a foreign PG server port from the catalog
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Foreign_Server_Port_PG(server_internal name)
RETURNS integer
AS $$
SELECT option_value::integer FROM (
SELECT (pg_options_to_table(srvoptions)).*
FROM pg_foreign_server WHERE srvname = server_internal
) AS opt WHERE opt.option_name = 'port';
$$
LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--
-- Get one measure of network latency in ms to a remote TCP server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(
server_internal name,
timeout_seconds float DEFAULT 5.0,
n_samples integer DEFAULT 10
)
RETURNS jsonb
AS $$
import socket
import json
import math
from timeit import default_timer as timer
plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Host_PG($1) AS host", ['name'])
rv = plpy.execute(plan, [server_internal], 1)
host = rv[0]['host']
plan = plpy.prepare("SELECT @extschema@.__CDB_FS_Foreign_Server_Port_PG($1) AS port", ['name'])
rv = plpy.execute(plan, [server_internal], 1)
port = rv[0]['port'] or 5432
n_errors = 0
samples = []
for i in range(n_samples):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout_seconds)
t_start = timer()
try:
s.connect((host, int(port)))
t_stop = timer()
s.shutdown(socket.SHUT_RD)
except (socket.timeout, OSError, socket.error) as ex:
plpy.warning('could not connect to server %s:%d, %s' % (host, port, str(ex)))
n_errors += 1
finally:
s.close()
t_connect = (t_stop - t_start) * 1000.0
plpy.debug('TCP connection %s:%d time=%.2f ms' % (host, port, t_connect))
samples.append(t_connect)
stats = {
'n_samples': n_samples,
'n_errors': n_errors,
}
n = len(samples)
if n >= 1:
mean = sum(samples) / n
stats.update({
'avg': round(mean, 3),
'min': round(min(samples), 3),
'max': round(max(samples), 3)
})
if n >= 2:
var = sum([ (x - mean)**2 for x in samples ]) / (n-1)
stdev = math.sqrt(var)
stats.update({
'stdev': round(stdev, 3)
})
return json.dumps(stats)
$$
LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
--
-- Collect and return diagnostics info from a remote PG into a jsonb
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal name)
RETURNS jsonb
AS $$
DECLARE
remote_server_version text := @extschema@.__CDB_FS_Foreign_Server_Version_PG(server_internal);
remote_postgis_version text := @extschema@.__CDB_FS_Foreign_PostGIS_Version_PG(server_internal);
remote_server_options jsonb := @extschema@.__CDB_FS_Foreign_Server_Options_PG(server_internal);
remote_server_latency_ms jsonb := @extschema@.__CDB_FS_TCP_Foreign_Server_Latency(server_internal);
BEGIN
RETURN jsonb_build_object(
'server_version', remote_server_version,
'postgis_version', remote_postgis_version,
'server_options', remote_server_options,
'server_latency_ms', remote_server_latency_ms
);
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Collect and return diagnostics info from a remote PG into a jsonb
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_Diagnostics(server TEXT)
RETURNS jsonb
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN @extschema@.__CDB_FS_Server_Diagnostics_PG(server_internal);
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

305
scripts-available/CDB_FederatedServerListRemote.sql
View File

@@ -1,7 +1,298 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Schemas_PG(name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Tables_PG(name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Columns_PG(name, name);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(name, name, name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Schemas(TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Tables(TEXT, TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Server_List_Remote_Columns(TEXT, TEXT, TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- List the schemas of a remote PG server
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal name)
RETURNS TABLE(remote_schema name)
AS $$
DECLARE
-- Import schemata from the information schema
--
-- "The view schemata contains all schemas in the current database
-- that the current user has access to (by way of being the owner
-- or having some privilege)."
-- See https://www.postgresql.org/docs/11/infoschema-schemata.html
--
-- "The information schema is defined in the SQL standard and can
-- therefore be expected to be portable and remain stable"
-- See https://www.postgresql.org/docs/11/information-schema.html
inf_schema name := 'information_schema';
remote_table name := 'schemata';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign schemata table
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_table, server_internal, local_schema);
END IF;
-- Return the result we're interested in. Exclude toast and temp schemas
BEGIN
RETURN QUERY EXECUTE format('
SELECT schema_name::name AS remote_schema FROM %I.%I
WHERE schema_name NOT LIKE %s
ORDER BY remote_schema
', local_schema, remote_table, '''pg_%''');
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Not enough permissions to access the server "%"',
@extschema@.__CDB_FS_Extract_Server_Name(server_internal);
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the names of the tables in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal name, remote_schema name)
RETURNS TABLE(remote_table name)
AS $func$
DECLARE
-- Import `tables` from the information schema
--
-- "The view tables contains all tables and views defined in the
-- current database. Only those tables and views are shown that
-- the current user has access to (by way of being the owner or
-- having some privilege)."
-- https://www.postgresql.org/docs/11/infoschema-tables.html
-- Create local target schema if it does not exists
inf_schema name := 'information_schema';
remote_table name := 'tables';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign `tables` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_table, server_internal, local_schema);
END IF;
-- Note: in this context, schema names are not to be quoted
RETURN QUERY EXECUTE format($q$
SELECT table_name::name AS remote_table FROM %I.%I WHERE table_schema = '%s' ORDER BY table_name
$q$, local_schema, remote_table, remote_schema);
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the columns in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal name, remote_schema name)
RETURNS TABLE(table_name name, column_name name, column_type text)
AS $func$
DECLARE
-- Import `columns` from the information schema
--
-- "The view columns contains information about all table columns (or view columns)
-- in the database. System columns (oid, etc.) are not included. Only those columns
-- are shown that the current user has access to (by way of being the owner or having some privilege)."
-- https://www.postgresql.org/docs/11/infoschema-columns.html
-- Create local target schema if it does not exists
inf_schema name := 'information_schema';
remote_col_table name := 'columns';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, inf_schema);
BEGIN
-- Import the foreign `columns` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_col_table
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I',
inf_schema, remote_col_table, server_internal, local_schema);
END IF;
-- Note: in this context, schema names are not to be quoted
-- We join with the geometry columns to change the type `USER-DEFINED`
-- by its appropiate geometry and srid
RETURN QUERY EXECUTE format($q$
SELECT
a.table_name::name,
a.column_name::name,
COALESCE(b.column_type, a.data_type)::TEXT as column_type
FROM
%I.%I a
LEFT JOIN
@extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG('%s', '%s') b
ON a.table_name = b.table_name AND a.column_name = b.column_name
WHERE table_schema = '%s'
ORDER BY a.table_name, a.column_name $q$,
local_schema, remote_col_table,
server_internal, remote_schema,
remote_schema);
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the geometry columns in a remote PG schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Foreign_Geometry_Columns_PG(server_internal name, remote_schema name, postgis_schema name DEFAULT 'public')
RETURNS TABLE(table_name name, column_name name, column_type text)
AS $func$
DECLARE
-- Import `geometry_columns` and `geography_columns` from the postgis schema
-- We assume that postgis is installed in the public schema
-- Create local target schema if it does not exists
remote_geometry_view name := 'geometry_columns';
remote_geography_view name := 'geography_columns';
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, postgis_schema);
BEGIN
-- Import the foreign `geometry_columns` and `geography_columns` if not done
IF NOT EXISTS (
SELECT * FROM pg_class
WHERE relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = local_schema)
AND relname = remote_geometry_view
) THEN
EXECUTE format('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I, %I) FROM SERVER %I INTO %I',
postgis_schema, remote_geometry_view, remote_geography_view, server_internal, local_schema);
END IF;
BEGIN
-- Note: We return both the type and srid as the type
RETURN QUERY EXECUTE format($q$
SELECT
f_table_name::NAME as table_name,
f_geometry_column::NAME as column_name,
type::TEXT || ',' || srid::TEXT as column_type
FROM
(
SELECT * FROM %I.%I UNION ALL SELECT * FROM %I.%I
) _geo_views
WHERE f_table_schema = '%s'
$q$,
local_schema, remote_geometry_view,
local_schema, remote_geography_view,
remote_schema);
EXCEPTION WHEN OTHERS THEN
RAISE INFO 'Could not find Postgis installation in the remote "%" schema in server "%"',
postgis_schema, @extschema@.__CDB_FS_Extract_Server_Name(server_internal);
RETURN;
END;
END
$func$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- List remote schemas in a federated server that the current user has access to.
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Schemas(server TEXT)
RETURNS TABLE(remote_schema name)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY SELECT @extschema@.__CDB_FS_List_Foreign_Schemas_PG(server_internal);
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List remote tables in a federated server that the current user has access to.
-- For registered tables it returns also the associated configuration
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Tables(server TEXT, remote_schema TEXT)
RETURNS TABLE(
registered boolean,
remote_table TEXT,
local_qualified_name TEXT,
id_column_name TEXT,
geom_column_name TEXT,
webmercator_column_name TEXT,
columns JSON
)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY
SELECT
coalesce(registered_tables.registered, false)::boolean as registered,
foreign_tables.remote_table::text as remote_table,
registered_tables.local_qualified_name as local_qualified_name,
registered_tables.id_column_name as id_column_name,
registered_tables.geom_column_name as geom_column_name,
registered_tables.webmercator_column_name as webmercator_column_name,
remote_columns.columns as columns
FROM
@extschema@.__CDB_FS_List_Foreign_Tables_PG(server_internal, remote_schema) foreign_tables
LEFT JOIN
@extschema@.__CDB_FS_List_Registered_Tables(server_internal, remote_schema) registered_tables
ON foreign_tables.remote_table = registered_tables.remote_table
LEFT JOIN
( -- Extract and group columns with their remote table
SELECT table_name,
json_agg(json_build_object('Name', column_name, 'Type', column_type)) as columns
FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
GROUP BY table_name
) remote_columns
ON foreign_tables.remote_table = remote_columns.table_name
ORDER BY foreign_tables.remote_table;
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- List the columns of a remote table in a federated server that the current user has access to.
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Server_List_Remote_Columns(
server TEXT,
remote_schema TEXT,
remote_table TEXT)
RETURNS TABLE(column_n name, column_t text)
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => true);
server_type name := @extschema@.__CDB_FS_server_type(server_internal);
BEGIN
IF remote_table IS NULL THEN
RAISE EXCEPTION 'Remote table name cannot be NULL';
END IF;
CASE server_type
WHEN 'postgres_fdw' THEN
RETURN QUERY
SELECT
column_name,
column_type
FROM @extschema@.__CDB_FS_List_Foreign_Columns_PG(server_internal, remote_schema)
WHERE table_name = remote_table
ORDER BY column_name;
ELSE
RAISE EXCEPTION 'Not implemented server type % for remote server %', server_type, remote_server;
END CASE;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

354
scripts-available/CDB_FederatedServerTables.sql
View File

@@ -1,9 +1,345 @@
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Integer(REGCLASS, NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Column_Is_Geometry(REGCLASS, NAME);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_GetColumns(REGCLASS);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_id_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_geom_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_Get_View_webmercator_column(TEXT);
DROP FUNCTION IF EXISTS @extschema@.__CDB_FS_List_Registered_Tables(NAME,TEXT);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Register(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, NAME);
DROP FUNCTION IF EXISTS @extschema@.CDB_Federated_Table_Unregister(TEXT, TEXT, TEXT);
--------------------------------------------------------------------------------
-- Private functions
--------------------------------------------------------------------------------
--
-- Checks if a column is of integer type
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Integer(input_table REGCLASS, colname NAME)
RETURNS boolean
AS $$
BEGIN
PERFORM atttypid FROM pg_catalog.pg_attribute
WHERE attrelid = input_table
AND attname = colname
AND atttypid IN (SELECT oid FROM pg_type
WHERE typname IN
('smallint', 'integer', 'bigint', 'int2', 'int4', 'int8'));
RETURN FOUND;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
--
-- Checks if a column is of geometry type
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Column_Is_Geometry(input_table REGCLASS, colname NAME)
RETURNS boolean
AS $$
BEGIN
PERFORM atttypid FROM pg_catalog.pg_attribute
WHERE attrelid = input_table
AND attname = colname
AND atttypid = '@postgisschema@.geometry'::regtype;
RETURN FOUND;
END
$$
LANGUAGE PLPGSQL STABLE PARALLEL UNSAFE;
--
-- Returns the name of all the columns from a table
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_GetColumns(input_table REGCLASS)
RETURNS SETOF NAME
AS $$
SELECT
a.attname as "colname"
FROM pg_catalog.pg_attribute a
WHERE
a.attnum > 0
AND NOT a.attisdropped
AND a.attrelid = (
SELECT c.oid
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.oid = input_table::oid
)
ORDER BY a.attnum;
$$ LANGUAGE SQL STABLE PARALLEL UNSAFE;
--
-- Returns the id column from a view definition
-- Note: The id is always of one of this forms:
-- SELECT t.cartodb_id,
-- SELECT t.my_id as cartodb_id,
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_id_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[2], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.cartodb_id', 'cartodb_id')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Returns the geom column from a view definition
--
-- Note: The the_geom is always of one of this forms:
-- t.the_geom,
-- t.my_geom as the_geom,
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_geom_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom', 'the_geom')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- Returns the webmercatorcolumn from a view definition
-- Note: The the_geom_webmercator is always of one of this forms:
-- t.the_geom_webmercator,
-- t.my_geom as the_geom_webmercator,
-- Or without the trailing comma:
-- t.the_geom_webmercator
-- t.my_geom as the_geom_webmercator
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_Get_View_webmercator_column(view_def TEXT)
RETURNS TEXT
AS $$
WITH column_definitions AS
(
SELECT regexp_split_to_array(trim(leading from regexp_split_to_table(view_def, '\n')), ' ') AS col_def
)
SELECT trim(trailing ',' FROM split_part(col_def[1], '.', 2))
FROM column_definitions
WHERE trim(trailing ',' FROM col_def[array_length(col_def, 1)]) IN ('t.the_geom_webmercator', 'the_geom_webmercator')
LIMIT 1;
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
--
-- List all registered tables in a server + schema
--
CREATE OR REPLACE FUNCTION @extschema@.__CDB_FS_List_Registered_Tables(
server_internal NAME,
remote_schema TEXT
)
RETURNS TABLE(
registered boolean,
remote_table TEXT,
local_qualified_name TEXT,
id_column_name TEXT,
geom_column_name TEXT,
webmercator_column_name TEXT
)
AS $$
DECLARE
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
RETURN QUERY SELECT
true as registered,
source_table::text as remote_table,
format('%I.%I', dependent_schema, dependent_view)::text as local_qualified_name,
@extschema@.__CDB_FS_Get_View_id_column(view_definition) as id_column_name,
@extschema@.__CDB_FS_Get_View_geom_column(view_definition) as geom_column_name,
@extschema@.__CDB_FS_Get_View_webmercator_column(view_definition) as webmercator_column_name
FROM
(
SELECT DISTINCT
dependent_ns.nspname as dependent_schema,
dependent_view.relname as dependent_view,
source_table.relname as source_table,
pg_get_viewdef(dependent_view.oid) as view_definition
FROM pg_depend
JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
JOIN pg_namespace dependent_ns ON dependent_ns.oid = dependent_view.relnamespace
JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
WHERE
source_ns.nspname = local_schema
ORDER BY 1,2
) _aux;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Public functions
--------------------------------------------------------------------------------
--
-- Sets up a Federated Table
--
-- Precondition: the federated server has to be set up via
-- CDB_Federated_Server_Register_PG
--
-- Postcondition: it generates a view in the schema of the user that
-- can be used through SQL and Maps API's.
-- If the table was already exported, it will be dropped and re-imported
--
-- The view is placed under the server's view schema (cdb_fs_$(server_name))
--
-- E.g:
-- SELECT cartodb.CDB_SetUp_PG_Federated_Table(
-- 'amazon', -- mandatory, name of the federated server
-- 'my_remote_schema', -- mandatory, schema name
-- 'my_remote_table', -- mandatory, table name
-- 'id', -- mandatory, name of the id column
-- 'geom', -- optional, name of the geom column, preferably in 4326
-- 'webmercator' -- optional, should be in 3857 if present
-- 'local_name' -- optional, name of the local view (uses the remote_name if not declared)
-- );
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Register(
server TEXT,
remote_schema TEXT,
remote_table TEXT,
id_column TEXT,
geom_column TEXT DEFAULT NULL,
webmercator_column TEXT DEFAULT NULL,
local_name NAME DEFAULT NULL
)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
role_name name := @extschema@.__CDB_FS_Generate_Server_Role_Name(server_internal);
src_table REGCLASS; -- import_schema.remote_table - import_schema is local_schema
local_view text; -- view_schema.local_name - view_schema is server_internal
rest_of_cols TEXT[];
geom_expression TEXT;
webmercator_expression TEXT;
carto_columns_expression TEXT[];
BEGIN
IF remote_table IS NULL THEN
RAISE EXCEPTION 'Remote table name cannot be NULL';
END IF;
-- Make do with whatever columns are provided
IF webmercator_column IS NULL THEN
webmercator_column := geom_column;
ELSIF geom_column IS NULL THEN
geom_column := webmercator_column;
END IF;
IF local_name IS NULL THEN
local_name := remote_table;
END IF;
-- Import the foreign table
-- Drop the old view / table if there was one
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = local_schema AND table_name = remote_table) THEN
EXECUTE @extschema@.CDB_Federated_Table_Unregister(server, remote_schema, remote_table);
END IF;
BEGIN
EXECUTE FORMAT('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;',
remote_schema, remote_table, server_internal, local_schema);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not import schema "%" of server "%": %', remote_schema, server, SQLERRM;
END;
BEGIN
src_table := format('%I.%I', local_schema, remote_table);
EXCEPTION WHEN OTHERS THEN
RAISE EXCEPTION 'Could not import table "%.%" of server "%"', remote_schema, remote_table, server;
END;
-- Check id_column is numeric
IF NOT @extschema@.__CDB_FS_Column_Is_Integer(src_table, id_column) THEN
RAISE EXCEPTION 'non integer id_column "%"', id_column;
END IF;
-- Check if the geom and mercator columns have a geometry type (if provided)
IF geom_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, geom_column) THEN
RAISE EXCEPTION 'non geometry column "%"', geom_column;
END IF;
IF webmercator_column IS NOT NULL AND NOT @extschema@.__CDB_FS_Column_Is_Geometry(src_table, webmercator_column) THEN
RAISE EXCEPTION 'non geometry column "%"', webmercator_column;
END IF;
-- Get a list of columns excluding the id, geom and the_geom_webmercator
SELECT ARRAY(
SELECT quote_ident(c) FROM @extschema@.__CDB_FS_GetColumns(src_table) AS c
WHERE c NOT IN (SELECT * FROM (SELECT unnest(ARRAY[id_column, geom_column, webmercator_column, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) col) carto WHERE carto.col IS NOT NULL)
) INTO rest_of_cols;
IF geom_column IS NULL
THEN
geom_expression := 'NULL AS the_geom';
ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, geom_column::varchar) = 4326
THEN
geom_expression := format('t.%I AS the_geom', geom_column);
ELSE
-- It needs an ST_Transform to 4326
geom_expression := format('@postgisschema@.ST_Transform(t.%I,4326) AS the_geom', geom_column);
END IF;
IF webmercator_column IS NULL
THEN
webmercator_expression := 'NULL AS the_geom_webmercator';
ELSIF @postgisschema@.Find_SRID(local_schema::varchar, remote_table::varchar, webmercator_column::varchar) = 3857
THEN
webmercator_expression := format('t.%I AS the_geom_webmercator', webmercator_column);
ELSE
-- It needs an ST_Transform to 3857
webmercator_expression := format('@postgisschema@.ST_Transform(t.%I,3857) AS the_geom_webmercator', webmercator_column);
END IF;
-- CARTO columns expressions
carto_columns_expression := ARRAY[
format('t.%1$I AS cartodb_id', id_column),
geom_expression,
webmercator_expression
];
-- Create view schema if it doesn't exist
IF NOT EXISTS (SELECT oid FROM pg_namespace WHERE nspname = server_internal) THEN
EXECUTE 'CREATE SCHEMA ' || quote_ident(server_internal) || ' AUTHORIZATION ' || quote_ident(role_name);
END IF;
-- Create a view with homogeneous CDB fields
BEGIN
EXECUTE format(
'CREATE OR REPLACE VIEW %1$I.%2$I AS
SELECT %3s
FROM %4$s t',
server_internal, local_name,
array_to_string(carto_columns_expression || rest_of_cols, ','),
src_table
);
EXCEPTION WHEN OTHERS THEN
IF EXISTS (SELECT to_regclass(format('%1$I.%2$I', server_internal, local_name))) THEN
RAISE EXCEPTION 'Could not import table "%" as "%.%" already exists', remote_table, server_internal, local_name;
ELSE
RAISE EXCEPTION 'Could not import table "%" as "%": %', remote_table, local_name, SQLERRM;
END IF;
END;
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--
-- Unregisters a remote table. Any dependent object will be dropped
--
CREATE OR REPLACE FUNCTION @extschema@.CDB_Federated_Table_Unregister(
server TEXT,
remote_schema TEXT,
remote_table TEXT
)
RETURNS void
AS $$
DECLARE
server_internal name := @extschema@.__CDB_FS_Generate_Server_Name(input_name => server, check_existence => false);
local_schema name := @extschema@.__CDB_FS_Create_Schema(server_internal, remote_schema);
BEGIN
EXECUTE FORMAT ('DROP FOREIGN TABLE %I.%I CASCADE;', local_schema, remote_table);
END
$$
LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;

179
scripts-available/CDB_ForeignTable.sql
View File

@@ -139,176 +139,6 @@ $$
LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- Produce a valid DB name for objects created for the user FDW's
CREATE OR REPLACE FUNCTION @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name NAME)
RETURNS NAME AS $$
-- Note on input we use %s and on output we use %I, in order to
-- avoid double escaping
SELECT format('cdb_fdw_%s', fdw_input_name)::NAME;
$$
LANGUAGE sql IMMUTABLE PARALLEL SAFE;
-- A function to set up a user-defined foreign data server
-- It does not read from CDB_Conf.
-- Only superuser roles can invoke it successfully
--
-- Sample call:
-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Server('amazon', '{
-- "server": {
-- "extensions": "postgis",
-- "dbname": "testdb",
-- "host": "myhostname.us-east-2.rds.amazonaws.com",
-- "port": "5432"
-- },
-- "user_mapping": {
-- "user": "fdw_user",
-- "password": "secret"
-- }
-- }');
--
-- Underneath it will:
-- * Set up postgresql_fdw
-- * Create a server with the name 'cdb_fdw_amazon'
-- * Create a role called 'cdb_fdw_amazon' to manage access
-- * Create a user mapping with that role 'cdb_fdw_amazon'
-- * Create a schema 'cdb_fdw_amazon' as a convenience to set up all foreign
-- tables over there
--
-- It is the responsibility of the superuser to grant that role to either:
-- * Nobody
-- * Specific roles: GRANT amazon TO role_name;
-- * Members of the organization: SELECT cartodb.CDB_Organization_Grant_Role('cdb_fdw_amazon');
-- * The publicuser: GRANT cdb_fdw_amazon TO publicuser;
CREATE OR REPLACE FUNCTION @extschema@._CDB_SetUp_User_PG_FDW_Server(fdw_input_name NAME, config json)
RETURNS void AS $$
DECLARE
row record;
option record;
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
BEGIN
-- TODO: refactor with original function
-- This function tries to be as idempotent as possible, by not creating anything more than once
-- (not even using IF NOT EXIST to avoid throwing warnings)
IF NOT EXISTS ( SELECT * FROM pg_extension WHERE extname = 'postgres_fdw') THEN
CREATE EXTENSION postgres_fdw;
RAISE NOTICE 'Created postgres_fdw EXTENSION';
END IF;
-- Create FDW first if it does not exist
IF NOT EXISTS ( SELECT * FROM pg_foreign_server WHERE srvname = fdw_objects_name)
THEN
EXECUTE FORMAT('CREATE SERVER %I FOREIGN DATA WRAPPER postgres_fdw', fdw_objects_name);
RAISE NOTICE 'Created SERVER % using postgres_fdw', fdw_objects_name;
END IF;
-- Set FDW settings
FOR row IN SELECT p.key, p.value from lateral json_each_text(config->'server') p
LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(srvoptions), '=', 1) as options from pg_foreign_server where srvname=fdw_objects_name) SELECT * from a where options = row.key)
THEN
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, row.key, row.value);
ELSE
EXECUTE FORMAT('ALTER SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, row.key, row.value);
END IF;
END LOOP;
-- Create specific role for this
IF NOT EXISTS ( SELECT 1 FROM pg_roles WHERE rolname = fdw_objects_name) THEN
EXECUTE format('CREATE ROLE %I NOLOGIN', fdw_objects_name);
RAISE NOTICE 'Created special ROLE % to access the correponding FDW', fdw_objects_name;
END IF;
-- Transfer ownership of the server to the fdw role
EXECUTE format('ALTER SERVER %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
-- Create user mapping
-- NOTE: we use a PUBLIC user mapping but control access to the SERVER
-- so that we don't need to create a mapping for every user nor store credentials elsewhere
IF NOT EXISTS ( SELECT * FROM pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public' ) THEN
EXECUTE FORMAT ('CREATE USER MAPPING FOR public SERVER %I', fdw_objects_name);
RAISE NOTICE 'Created USER MAPPING for accesing foreign server %', fdw_objects_name;
END IF;
-- Update user mapping settings
FOR option IN SELECT o.key, o.value from lateral json_each_text(config->'user_mapping') o LOOP
IF NOT EXISTS (WITH a AS (select split_part(unnest(umoptions), '=', 1) as options from pg_user_mappings WHERE srvname = fdw_objects_name AND usename = 'public') SELECT * from a where options = option.key) THEN
EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (ADD %I %L)', fdw_objects_name, option.key, option.value);
ELSE
EXECUTE FORMAT('ALTER USER MAPPING FOR PUBLIC SERVER %I OPTIONS (SET %I %L)', fdw_objects_name, option.key, option.value);
END IF;
END LOOP;
-- Grant usage on the wrapper and server to the fdw role
EXECUTE FORMAT ('GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO %I', fdw_objects_name);
RAISE NOTICE 'Granted USAGE on the postgres_fdw to the role %', fdw_objects_name;
EXECUTE FORMAT ('GRANT USAGE ON FOREIGN SERVER %I TO %I', fdw_objects_name, fdw_objects_name);
RAISE NOTICE 'Granted USAGE on the foreign server to the role %', fdw_objects_name;
-- Create schema if it does not exist.
IF NOT EXISTS ( SELECT * from pg_namespace WHERE nspname=fdw_objects_name) THEN
EXECUTE FORMAT ('CREATE SCHEMA %I', fdw_objects_name);
RAISE NOTICE 'Created SCHEMA % to host foreign tables', fdw_objects_name;
END IF;
-- Give the fdw role ownership over the schema
EXECUTE FORMAT ('ALTER SCHEMA %I OWNER TO %I', fdw_objects_name, fdw_objects_name);
RAISE NOTICE 'Gave ownership on the SCHEMA % to %', fdw_objects_name, fdw_objects_name;
-- TODO: Bring here the remote cdb_tablemetadata
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- A function to drop a user-defined foreign server and all related objects
-- It does not read from CDB_Conf
-- It must be executed with a superuser role to succeed
--
-- Sample call:
-- SELECT cartodb.CDB_Drop_User_PG_FDW_Server('amazon')
--
-- Note: if there's any dependent object (i.e. foreign table) this call will fail
CREATE OR REPLACE FUNCTION @extschema@._CDB_Drop_User_PG_FDW_Server(fdw_input_name NAME, force boolean = false)
RETURNS void AS $$
DECLARE
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
cascade_clause NAME;
BEGIN
CASE force
WHEN true THEN
cascade_clause := 'CASCADE';
ELSE
cascade_clause := 'RESTRICT';
END CASE;
EXECUTE FORMAT ('DROP SCHEMA %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Dropped schema %', fdw_objects_name;
EXECUTE FORMAT ('DROP USER MAPPING FOR public SERVER %I', fdw_objects_name);
RAISE NOTICE 'Dropped user mapping for server %', fdw_objects_name;
EXECUTE FORMAT ('DROP SERVER %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Dropped foreign server %', fdw_objects_name;
EXECUTE FORMAT ('REVOKE USAGE ON FOREIGN DATA WRAPPER postgres_fdw FROM %I %s', fdw_objects_name, cascade_clause);
RAISE NOTICE 'Revoked usage on postgres_fdw from %', fdw_objects_name;
EXECUTE FORMAT ('DROP ROLE %I', fdw_objects_name);
RAISE NOTICE 'Dropped role %', fdw_objects_name;
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
-- Set up a user foreign table
-- E.g:
-- SELECT cartodb.CDB_SetUp_User_PG_FDW_Table('amazon', 'carto_lite', 'mytable');
-- SELECT * FROM amazon.my_table;
CREATE OR REPLACE FUNCTION @extschema@.CDB_SetUp_User_PG_FDW_Table(fdw_input_name NAME, foreign_schema NAME, table_name NAME)
RETURNS void AS $$
DECLARE
fdw_objects_name NAME := @extschema@.__CDB_User_FDW_Object_Names(fdw_input_name);
BEGIN
EXECUTE FORMAT ('IMPORT FOREIGN SCHEMA %I LIMIT TO (%I) FROM SERVER %I INTO %I;', foreign_schema, table_name, fdw_objects_name, fdw_objects_name);
--- Grant SELECT to fdw role
EXECUTE FORMAT ('GRANT SELECT ON %I.%I TO %I;', fdw_objects_name, table_name, fdw_objects_name);
END
$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION @extschema@._cdb_dbname_of_foreign_table(reloid oid)
RETURNS TEXT AS $$
SELECT option_value FROM pg_options_to_table((
@@ -374,3 +204,12 @@ RETURNS timestamptz AS $$
LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid
) SELECT max(updated_at) FROM t_updated_at;
$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Deprecated
--------------------------------------------------------------------------------
DROP FUNCTION IF EXISTS @extschema@.__CDB_User_FDW_Object_Names(name);
DROP FUNCTION IF EXISTS @extschema@._CDB_SetUp_User_PG_FDW_Server(name, json);
DROP FUNCTION IF EXISTS @extschema@._CDB_Drop_User_PG_FDW_Server(name, boolean);
DROP FUNCTION IF EXISTS @extschema@.CDB_SetUp_User_PG_FDW_Table(name, name, name);

1
scripts-available/CDB_Groups_API.sql
View File

@@ -5,7 +5,6 @@
-- Requires configuration parameter. Example: SELECT @extschema@.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
----------------------------------
-- TODO: delete this development cleanup before final merge
DROP FUNCTION IF EXISTS @extschema@.CDB_Group_AddMember(group_name text, username text);
DROP FUNCTION IF EXISTS @extschema@.CDB_Group_RemoveMember(group_name text, username text);
DROP FUNCTION IF EXISTS @extschema@._CDB_Group_AddMember_API(group_name text, username text);

26
scripts-available/CDB_Organizations.sql
View File

@@ -172,27 +172,7 @@ $$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
--------------------------------------------------------------------------------
-- Role management
-- Deprecated
--------------------------------------------------------------------------------
CREATE OR REPLACE
FUNCTION @extschema@.CDB_Organization_Grant_Role(role_name name)
RETURNS VOID AS $$
DECLARE
org_role TEXT;
BEGIN
org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
EXECUTE format('GRANT %I TO %I', role_name, org_role);
END
$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE
FUNCTION @extschema@.CDB_Organization_Revoke_Role(role_name name)
RETURNS VOID AS $$
DECLARE
org_role TEXT;
BEGIN
org_role := @extschema@.CDB_Organization_Member_Group_Role_Member_Name();
EXECUTE format('REVOKE %I FROM %I', role_name, org_role);
END
$$ LANGUAGE PLPGSQL VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Grant_Role(name);
DROP FUNCTION IF EXISTS @extschema@.CDB_Organization_Revoke_Role(name);