diff --git a/README.md b/README.md index 2f8af0a..7fe5f8e 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Steps to deploy a new Data Services API version : - install python library ``` - cd server/lib/python/cartodb_services && python setup.py install + cd server/lib/python/cartodb_services && sudo pip install --upgrade . ``` - install extensions in user database @@ -62,16 +62,16 @@ Steps to deploy a new Data Services API version : create extension cdb_dataservices_client; ``` -- add configuration for different services in user database +- add configuration for different services in server database ``` # If sentinel is used: SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); # If sentinel is not used - SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}'); @@ -81,10 +81,13 @@ Steps to deploy a new Data Services API version : SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}') ``` -- configure plproxy to point to the a database (you can use a specific database for the server or your same user) +- configure the user DB: - ``` - SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); + ```sql + -- Point to the dataservices server DB (you can use a specific database for the server or your same user's): + SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); + + SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); ``` - configure the search path in order to be able to execute the functions without using the schema: diff --git a/client/sql/20_table_augmentation.sql b/client/sql/20_table_augmentation.sql new file mode 100644 index 0000000..8b6cf09 --- /dev/null +++ b/client/sql/20_table_augmentation.sql @@ -0,0 +1,263 @@ +CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text); +CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json) +RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json) +RETURNS boolean AS $$ +DECLARE + username text; + user_db_role text; + orgname text; + dbname text; + user_schema text; + result boolean; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + + SELECT session_user INTO user_db_role; + + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument'; + END IF; + + IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN + user_schema := 'public'; + ELSE + user_schema := username; + END IF; + + SELECT current_database() INTO dbname; + + SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result; + + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json) +RETURNS boolean AS $$ + from time import strftime + try: + server_table_name = None + temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name + + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) + ) + + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + + # Prepare column and type strings required in the SQL queries + colnames = ','.join(colnames_arr) + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) + ) + + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + + # Create temporary table with the augmented results + plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS ' + '(SELECT {columns}, cartodb_id ' + 'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData(' + '{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) ' + 'AS results({columns_with_types}, cartodb_id int) )' + .format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), + user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name), + function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types, + temp_table_name=temporary_table_name) + ) + + # Wipe user FDW data from the server + wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) + ) + + # Prepare table to receive augmented results in new columns + for idx, column in enumerate(colnames_arr): + if colnames_arr[idx] is not 'the_geom': + plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}' + .format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx]) + ) + + # Populate the user table with the augmented results + plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = ' + '(SELECT {columns} FROM "{user_schema}".{temporary_table_name} ' + 'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)' + .format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), + user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types, + temporary_table_name=temporary_table_name) + ) + + plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' + .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) + ) + + return True + except Exception as e: + plpy.warning('Error trying to augment table {0}'.format(e)) + # Wipe user FDW data from the server in case of failure if the table was connected + if server_table_name: + # Wipe local temporary table + plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}' + .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name) + ) + + wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) + ) + return False +$$ LANGUAGE plpythonu; + + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json) +RETURNS boolean AS $$ + try: + server_table_name = None + # Obtain return types for augmentation procedure + ds_return_metadata = plpy.execute("SELECT colnames, coltypes " + "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params)) + ) + + colnames_arr = ds_return_metadata[0]["colnames"] + coltypes_arr = ds_return_metadata[0]["coltypes"] + + # Prepare column and type strings required in the SQL queries + colnames = ','.join(colnames_arr) + columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))] + columns_with_types = ','.join(columns_with_types_arr) + + + # Instruct the OBS server side to establish a FDW + # The metadata is obtained as well in order to: + # - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy, + # - (b) be able to tell OBS to free resources when done. + ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername " + "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) + ) + + server_schema = ds_fdw_metadata[0]["schemaname"] + server_table_name = ds_fdw_metadata[0]["tabname"] + server_name = ds_fdw_metadata[0]["servername"] + + # Get list of user columns to include in the new table + user_table_columns = ','.join( + plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns ' + 'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass ' + 'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' ' + 'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));' + .format(user_schema=user_schema, table_name=table_name, colnames=colnames) + )[0]["columns"] + ) + + # Populate a new table with the augmented results + plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS ' + '(SELECT results.{columns}, {user_table_columns} ' + 'FROM {table_name} AS user_table ' + 'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) ' + 'ON results.cartodb_id = user_table.cartodb_id)' + .format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username), + orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), + table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types) + ) + + plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";' + .format(schema=user_schema, table_name=output_table_name, user=user_db_role) + ) + + # Wipe user FDW data from the server + wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) + ) + + return True + except Exception as e: + plpy.warning('Error trying to get table {0}'.format(e)) + # Wipe user FDW data from the server in case of failure if the table was connected + if server_table_name: + wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name)) + ) + return False +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ + CONNECT _server_conn_str(); + TARGET cdb_dataservices_server._OBS_ConnectUserTable; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_client.ds_return_metadata AS $$ + CONNECT _server_conn_str(); + TARGET cdb_dataservices_server._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT _server_conn_str(); + TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text) +RETURNS boolean AS $$ + CONNECT _server_conn_str(); + TARGET cdb_dataservices_server._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy; diff --git a/client/sql/95_grant_execute_manual.sql b/client/sql/95_grant_execute_manual.sql new file mode 100644 index 0000000..12047b1 --- /dev/null +++ b/client/sql/95_grant_execute_manual.sql @@ -0,0 +1,2 @@ +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser; diff --git a/client/test/expected/95_data_observatory_tables_test.out b/client/test/expected/95_data_observatory_tables_test.out new file mode 100644 index 0000000..faf311f --- /dev/null +++ b/client/test/expected/95_data_observatory_tables_test.out @@ -0,0 +1,67 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +CREATE TABLE my_table(cartodb_id int); +INSERT INTO my_table (cartodb_id) VALUES (1); +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ +BEGIN + RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text); +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_client.ds_return_metadata AS $$ +BEGIN + RETURN (Array['total_pop'], Array['double precision']); +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS RECORD AS $$ +BEGIN + RETURN (23.4::double precision, 1::int); +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ +BEGIN + RETURN true; +END; +$$ LANGUAGE 'plpgsql'; +-- Augment a table with the total_pop column +SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json); + _obs_augmenttable +------------------- + t +(1 row) + +-- The results of the table should return the mocked value of 23.4 in the total_pop column +SELECT * FROM my_table; + cartodb_id | total_pop +------------+----------- + 1 | 23.4 +(1 row) + +-- Mock again the function for it to return a different value now +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS RECORD AS $$ +BEGIN + RETURN (577777.4::double precision, 1::int); +END; +$$ LANGUAGE 'plpgsql'; +-- Augment a new table with total_pop +SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json); + _obs_gettable +--------------- + t +(1 row) + +-- Check that the table contains the new value for total_pop and not the value already existent in the table +SELECT * FROM my_table_new; + total_pop | cartodb_id +-----------+------------ + 577777.4 | 1 +(1 row) + +-- Clean tables +DROP TABLE my_table; +DROP TABLE my_table_new; diff --git a/client/test/sql/95_data_observatory_tables_test.sql b/client/test/sql/95_data_observatory_tables_test.sql new file mode 100644 index 0000000..717aea9 --- /dev/null +++ b/client/test/sql/95_data_observatory_tables_test.sql @@ -0,0 +1,59 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +CREATE TABLE my_table(cartodb_id int); + +INSERT INTO my_table (cartodb_id) VALUES (1); + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$ +BEGIN + RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text); +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_client.ds_return_metadata AS $$ +BEGIN + RETURN (Array['total_pop'], Array['double precision']); +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS RECORD AS $$ +BEGIN + RETURN (23.4::double precision, 1::int); +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ +BEGIN + RETURN true; +END; +$$ LANGUAGE 'plpgsql'; + +-- Augment a table with the total_pop column +SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json); + +-- The results of the table should return the mocked value of 23.4 in the total_pop column +SELECT * FROM my_table; + +-- Mock again the function for it to return a different value now +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS RECORD AS $$ +BEGIN + RETURN (577777.4::double precision, 1::int); +END; +$$ LANGUAGE 'plpgsql'; + +-- Augment a new table with total_pop +SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json); + +-- Check that the table contains the new value for total_pop and not the value already existent in the table +SELECT * FROM my_table_new; + +-- Clean tables +DROP TABLE my_table; +DROP TABLE my_table_new; \ No newline at end of file diff --git a/server/extension/old_versions/cdb_dataservices_server--0.11.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.11.0.sql index fac8e80..f9937be 100644 --- a/server/extension/old_versions/cdb_dataservices_server--0.11.0.sql +++ b/server/extension/old_versions/cdb_dataservices_server--0.11.0.sql @@ -2131,4 +2131,4 @@ BEGIN GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; GRANT USAGE ON SCHEMA public TO geocoder_api; GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; -END$$; +END$$; \ No newline at end of file diff --git a/server/extension/sql/125_data_observatory_table_augment.sql b/server/extension/sql/125_data_observatory_table_augment.sql new file mode 100644 index 0000000..963749a --- /dev/null +++ b/server/extension/sql/125_data_observatory_table_augment.sql @@ -0,0 +1,35 @@ +CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text); + +CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)" + .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name)) + )[0] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text) +RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_ConnectUserTable; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json) +RETURNS cdb_dataservices_server.ds_return_metadata AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_GetReturnMetadata; +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json) +RETURNS SETOF record AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_FetchJoinFdwTableData; +$$ LANGUAGE plproxy; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text) +RETURNS boolean AS $$ + CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname); + TARGET cdb_observatory._OBS_DisconnectUserTable; +$$ LANGUAGE plproxy; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 647f309..4c25de5 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -1,6 +1,5 @@ -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; CREATE EXTENSION plpythonu; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; diff --git a/server/extension/test/expected/100_data_observatory_tables_test.out b/server/extension/test/expected/100_data_observatory_tables_test.out new file mode 100644 index 0000000..6a3a063 --- /dev/null +++ b/server/extension/test/expected/100_data_observatory_tables_test.out @@ -0,0 +1,44 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_connectusertable' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_getreturnmetadata' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_fetchjoinfdwtabledata' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_disconnectusertable' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index e987940..0115c07 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -1,6 +1,5 @@ -- Install dependencies CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; CREATE EXTENSION plpythonu; CREATE EXTENSION plproxy; CREATE EXTENSION cartodb; diff --git a/server/extension/test/sql/100_data_observatory_tables_test.sql b/server/extension/test/sql/100_data_observatory_tables_test.sql new file mode 100644 index 0000000..cb63e1a --- /dev/null +++ b/server/extension/test/sql/100_data_observatory_tables_test.sql @@ -0,0 +1,28 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_connectusertable' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_getreturnmetadata' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_fetchjoinfdwtabledata' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, json'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_obs_disconnectusertable' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text'); +