From 0c43fe27311443df91d804c0ad2531f4f6ca7607 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 12:09:29 +0100 Subject: [PATCH 01/15] Define API of CDB_QueryTablesUpdatedAt --- scripts-available/CDB_QueryTables.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index c7cfa64..aaa3f8c 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -76,3 +76,15 @@ BEGIN RETURN CDB_QueryTablesText(query)::name[]; END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; + + +-- Return a set of {db_name, schema_name, table_name. updated_at} +CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) +RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamp) +AS $$ +BEGIN + -- TODO: Get the tables involved in the query + -- TODO: Get the local/remote db_names involved in the query + -- TODO: Get the updated_at +END +$$ LANGUAGE 'plpgsql' VOLATILE STRICT; From c2100081847b42f1978f2702718a87975fc54a71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 15:54:42 +0100 Subject: [PATCH 02/15] Skeleton of a possible solution (WIP) --- scripts-available/CDB_QueryTables.sql | 60 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index aaa3f8c..2862239 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -77,14 +77,70 @@ BEGIN END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; - +-------------------------------------------------------------------------------- -- Return a set of {db_name, schema_name, table_name. updated_at} CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamp) AS $$ +DECLARE + qualified_table_names text[]; + qualified_table_name text; + ret RECORD; BEGIN - -- TODO: Get the tables involved in the query + -- Get the tables involved in the query + SELECT CDB_QueryTablesText(query) INTO qualified_table_names; + + FOREACH qualified_table_name IN ARRAY qualified_table_names LOOP + --ret.db_name := 'db_name'; + RAISE DEBUG 'hola'; + END LOOP; + + RETURN QUERY + WITH qt AS (SELECT unnest(CDB_QueryTablesText(query)) qualified_table_name) + SELECT 'db_name'::text AS db_name, 'schema_name'::text AS schema_name, qt.qualified_table_name::text AS table_name, now()::timestamp AS udpated_at + FROM qt; + + -- TODO: Get the local/remote db_names involved in the query -- TODO: Get the updated_at END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; + + +-- Take a text containing "schema_name"."table_name" as input and +-- return a record of the form (db_name text, schema_name text, table_name text) +CREATE OR REPLACE FUNCTION _cdb_fqtn_from_text(schema_table_name text) +RETURNS RECORD AS $$ +DECLARE + ret RECORD; + reloid oid; + db_name text; + schema_name text; + table_name text; +BEGIN + SELECT schema_table_name::regclass INTO STRICT reloid; + + -- TODO: get if the table is local or remote + SELECT + CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(reloid) + ELSE current_database() + END as dbname, + n.nspname schema_name, c.relname table_name + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = reloid; + + + SELECT 'my_db_name'::text, 'my_schema_name'::text, 'my_table_name'::text INTO ret; + RETURN ret; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION _cdb_dbname_of_foreign_table(reloid oid) +RETURNS TEXT AS $$ +BEGIN + --TODO: implement + RETURN 'cartodb_dev_user_36c4a45a-eb92-4af4-a8ff-1065ecfd041f_db'; +END; +$$ LANGUAGE plpgsql; From 1e3c7ace9959e3128e41f4a1c14a50b28c9c78e9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 16:19:20 +0100 Subject: [PATCH 03/15] Implementation of _cdb_dbname_of_foreign_table (WIP) --- scripts-available/CDB_QueryTables.sql | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 2862239..68a702a 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -78,6 +78,7 @@ END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -------------------------------------------------------------------------------- + -- Return a set of {db_name, schema_name, table_name. updated_at} CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamp) @@ -139,8 +140,12 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION _cdb_dbname_of_foreign_table(reloid oid) RETURNS TEXT AS $$ -BEGIN - --TODO: implement - RETURN 'cartodb_dev_user_36c4a45a-eb92-4af4-a8ff-1065ecfd041f_db'; -END; -$$ LANGUAGE plpgsql; + SELECT option_value FROM pg_options_to_table(( + + SELECT fs.srvoptions + FROM pg_foreign_table ft + LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid + WHERE ft.ftrelid = reloid + + )) WHERE option_name='dbname'; +$$ LANGUAGE SQL; From 78a75cf22dcb3788b94600c07c0b761f8e36e06a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 16:38:04 +0100 Subject: [PATCH 04/15] Implementation of _cdb_fqtn_from_text (WIP) --- scripts-available/CDB_QueryTables.sql | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 68a702a..442a6ae 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -109,30 +109,26 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -- Take a text containing "schema_name"."table_name" as input and --- return a record of the form (db_name text, schema_name text, table_name text) +-- return a record of the form (dbname text, schema_name text, table_name text) CREATE OR REPLACE FUNCTION _cdb_fqtn_from_text(schema_table_name text) RETURNS RECORD AS $$ DECLARE - ret RECORD; reloid oid; - db_name text; - schema_name text; - table_name text; + ret RECORD; BEGIN SELECT schema_table_name::regclass INTO STRICT reloid; - -- TODO: get if the table is local or remote SELECT - CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(reloid) + (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(reloid) ELSE current_database() - END as dbname, - n.nspname schema_name, c.relname table_name + END)::text AS dbname, + n.nspname::text schema_name, + c.relname::text table_name + INTO ret FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid WHERE c.oid = reloid; - - SELECT 'my_db_name'::text, 'my_schema_name'::text, 'my_table_name'::text INTO ret; RETURN ret; END; $$ LANGUAGE plpgsql; From 1a12fd3b69b16094c7a1a7a470969854f865bd3a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 18:42:40 +0100 Subject: [PATCH 05/15] Make _cdb_fqtn_from_text return a table with type instead of a record --- scripts-available/CDB_QueryTables.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 442a6ae..5361572 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -111,25 +111,21 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -- Take a text containing "schema_name"."table_name" as input and -- return a record of the form (dbname text, schema_name text, table_name text) CREATE OR REPLACE FUNCTION _cdb_fqtn_from_text(schema_table_name text) -RETURNS RECORD AS $$ +RETURNS TABLE(dbname text, schema_name text, table_name text) AS $$ DECLARE reloid oid; - ret RECORD; BEGIN SELECT schema_table_name::regclass INTO STRICT reloid; - SELECT + RETURN QUERY SELECT (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(reloid) ELSE current_database() END)::text AS dbname, n.nspname::text schema_name, c.relname::text table_name - INTO ret FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid WHERE c.oid = reloid; - - RETURN ret; END; $$ LANGUAGE plpgsql; From a074f4df5d39f48f62ad602035cbd55afbfd3d36 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 18:43:25 +0100 Subject: [PATCH 06/15] Adapt CDB_QueryTablesUpdatedAt to _cdb_fqtn_from_text --- scripts-available/CDB_QueryTables.sql | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 5361572..5a50e64 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -79,30 +79,22 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -------------------------------------------------------------------------------- --- Return a set of {db_name, schema_name, table_name. updated_at} +-- Return a set of {dbname, schema_name, table_name. updated_at} CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamp) AS $$ -DECLARE - qualified_table_names text[]; - qualified_table_name text; - ret RECORD; BEGIN - -- Get the tables involved in the query - SELECT CDB_QueryTablesText(query) INTO qualified_table_names; - - FOREACH qualified_table_name IN ARRAY qualified_table_names LOOP - --ret.db_name := 'db_name'; - RAISE DEBUG 'hola'; - END LOOP; RETURN QUERY - WITH qt AS (SELECT unnest(CDB_QueryTablesText(query)) qualified_table_name) - SELECT 'db_name'::text AS db_name, 'schema_name'::text AS schema_name, qt.qualified_table_name::text AS table_name, now()::timestamp AS udpated_at - FROM qt; + WITH query_tables AS ( + SELECT unnest(CDB_QueryTablesText(query)) schema_table_name + ), fqtn AS ( + SELECT (_cdb_fqtn_from_text(schema_table_name)).* + FROM query_tables + ) + SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, now()::timestamp AS udpated_at + FROM fqtn; - - -- TODO: Get the local/remote db_names involved in the query -- TODO: Get the updated_at END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; From 0f21db51b622a725a2e8dcf548d52e5dda733567 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 19:24:09 +0100 Subject: [PATCH 07/15] Simplify the code and get the updated_at --- scripts-available/CDB_QueryTables.sql | 48 +++++++++++---------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 5a50e64..ae480e8 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -81,47 +81,39 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -- Return a set of {dbname, schema_name, table_name. updated_at} CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) -RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamp) +RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) AS $$ BEGIN RETURN QUERY WITH query_tables AS ( SELECT unnest(CDB_QueryTablesText(query)) schema_table_name - ), fqtn AS ( - SELECT (_cdb_fqtn_from_text(schema_table_name)).* + ), query_tables_oid AS ( + SELECT schema_table_name, schema_table_name::regclass::oid AS reloid FROM query_tables + ), + fqtn AS ( + SELECT + (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(query_tables_oid.reloid) + ELSE current_database() + END)::text AS dbname, + n.nspname::text schema_name, + c.relname::text table_name, + c.relkind, + query_tables_oid.reloid + FROM query_tables_oid, pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = query_tables_oid.reloid ) - SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, now()::timestamp AS udpated_at + SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, + (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE tabname = reloid) + END) AS updated_at FROM fqtn; - - -- TODO: Get the updated_at END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; --- Take a text containing "schema_name"."table_name" as input and --- return a record of the form (dbname text, schema_name text, table_name text) -CREATE OR REPLACE FUNCTION _cdb_fqtn_from_text(schema_table_name text) -RETURNS TABLE(dbname text, schema_name text, table_name text) AS $$ -DECLARE - reloid oid; -BEGIN - SELECT schema_table_name::regclass INTO STRICT reloid; - - RETURN QUERY SELECT - (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(reloid) - ELSE current_database() - END)::text AS dbname, - n.nspname::text schema_name, - c.relname::text table_name - FROM pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid - WHERE c.oid = reloid; -END; -$$ LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION _cdb_dbname_of_foreign_table(reloid oid) RETURNS TEXT AS $$ SELECT option_value FROM pg_options_to_table(( From ec34b8ee287a8b8db9f9ece11652ce471c1875ca Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Feb 2016 19:28:57 +0100 Subject: [PATCH 08/15] Minor changes: use plain SQL func --- scripts-available/CDB_QueryTables.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index ae480e8..418c649 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -83,9 +83,6 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) AS $$ -BEGIN - - RETURN QUERY WITH query_tables AS ( SELECT unnest(CDB_QueryTablesText(query)) schema_table_name ), query_tables_oid AS ( @@ -107,11 +104,10 @@ BEGIN ) SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) - ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE tabname = reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) END) AS updated_at FROM fqtn; -END -$$ LANGUAGE 'plpgsql' VOLATILE STRICT; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION _cdb_dbname_of_foreign_table(reloid oid) From 276b5cf9ea31431c139fb32e653fd02395bf3acb Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 13:29:04 +0100 Subject: [PATCH 09/15] Rename func to CDB_QueryTables_Updated_At s/CDB_QueryTablesUpdatedAt/CDB_QueryTables_Updated_At and also add a bit more inline doc. --- scripts-available/CDB_QueryTables.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 418c649..0abeb59 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -79,8 +79,9 @@ $$ LANGUAGE 'plpgsql' VOLATILE STRICT; -------------------------------------------------------------------------------- --- Return a set of {dbname, schema_name, table_name. updated_at} -CREATE OR REPLACE FUNCTION CDB_QueryTablesUpdatedAt(query text) +-- Return a set of (dbname, schema_name, table_name, updated_at) +-- This assumes the local (schema_name, table_name) maps to the remote one with the same name +CREATE OR REPLACE FUNCTION CDB_QueryTables_Updated_At(query text) RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) AS $$ WITH query_tables AS ( From eb84dd04c9d842f6293a6c4381e20152b808f121 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 13:30:00 +0100 Subject: [PATCH 10/15] Add func CDB_Last_Updated_Time --- scripts-available/CDB_QueryTables.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 0abeb59..80ef620 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -122,3 +122,22 @@ RETURNS TEXT AS $$ )) WHERE option_name='dbname'; $$ LANGUAGE SQL; + + +-- TODO: move to the right place +-- Return the last updated time of a set of tables +CREATE OR REPLACE FUNCTION CDB_Last_Updated_Time(tables text[]) +RETURNS timestamptz AS $$ + WITH t AS ( + SELECT unnest(tables) AS schema_table_name + ), t_oid AS ( + SELECT (t.schema_table_name)::regclass::oid as reloid FROM t + ), t_updated_at AS ( + SELECT + (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) + END) AS updated_at + FROM t_oid + LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid + ) SELECT max(updated_at) FROM t_updated_at; +$$ LANGUAGE SQL; From ecbdb4a430343099482b6fee0543ea6cb6f6ccaf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 13:40:18 +0100 Subject: [PATCH 11/15] Move fdw-aware functions to CDB_ForeignTable.sql --- scripts-available/CDB_ForeignTable.sql | 66 ++++++++++++++++++++++++++ scripts-available/CDB_QueryTables.sql | 65 ------------------------- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 85042db..6367512 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -130,3 +130,69 @@ BEGIN END $$ LANGUAGE plpgsql; + + +-- Return a set of (dbname, schema_name, table_name, updated_at) +-- It is aware of foreign tables +-- It assumes the local (schema_name, table_name) map to the remote ones with the same name +CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text) +RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) +AS $$ + WITH query_tables AS ( + SELECT unnest(CDB_QueryTablesText(query)) schema_table_name + ), query_tables_oid AS ( + SELECT schema_table_name, schema_table_name::regclass::oid AS reloid + FROM query_tables + ), + fqtn AS ( + SELECT + (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(query_tables_oid.reloid) + ELSE current_database() + END)::text AS dbname, + n.nspname::text schema_name, + c.relname::text table_name, + c.relkind, + query_tables_oid.reloid + FROM query_tables_oid, pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid + WHERE c.oid = query_tables_oid.reloid + ) + SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, + (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) + END) AS updated_at + FROM fqtn; +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid) +RETURNS TEXT AS $$ + SELECT option_value FROM pg_options_to_table(( + + SELECT fs.srvoptions + FROM pg_foreign_table ft + LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid + WHERE ft.ftrelid = reloid + + )) WHERE option_name='dbname'; +$$ LANGUAGE SQL; + + +-- Return the last updated time of a set of tables +-- It is aware of foreign tables +-- It assumes the local (schema_name, table_name) map to the remote ones with the same name +CREATE OR REPLACE FUNCTION cartodb.CDB_Last_Updated_Time(tables text[]) +RETURNS timestamptz AS $$ + WITH t AS ( + SELECT unnest(tables) AS schema_table_name + ), t_oid AS ( + SELECT (t.schema_table_name)::regclass::oid as reloid FROM t + ), t_updated_at AS ( + SELECT + (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) + END) AS updated_at + FROM t_oid + LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid + ) SELECT max(updated_at) FROM t_updated_at; +$$ LANGUAGE SQL; diff --git a/scripts-available/CDB_QueryTables.sql b/scripts-available/CDB_QueryTables.sql index 80ef620..c7cfa64 100644 --- a/scripts-available/CDB_QueryTables.sql +++ b/scripts-available/CDB_QueryTables.sql @@ -76,68 +76,3 @@ BEGIN RETURN CDB_QueryTablesText(query)::name[]; END $$ LANGUAGE 'plpgsql' VOLATILE STRICT; - --------------------------------------------------------------------------------- - --- Return a set of (dbname, schema_name, table_name, updated_at) --- This assumes the local (schema_name, table_name) maps to the remote one with the same name -CREATE OR REPLACE FUNCTION CDB_QueryTables_Updated_At(query text) -RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) -AS $$ - WITH query_tables AS ( - SELECT unnest(CDB_QueryTablesText(query)) schema_table_name - ), query_tables_oid AS ( - SELECT schema_table_name, schema_table_name::regclass::oid AS reloid - FROM query_tables - ), - fqtn AS ( - SELECT - (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(query_tables_oid.reloid) - ELSE current_database() - END)::text AS dbname, - n.nspname::text schema_name, - c.relname::text table_name, - c.relkind, - query_tables_oid.reloid - FROM query_tables_oid, pg_catalog.pg_class c - LEFT JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid - WHERE c.oid = query_tables_oid.reloid - ) - SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, - (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) - ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) - END) AS updated_at - FROM fqtn; -$$ LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION _cdb_dbname_of_foreign_table(reloid oid) -RETURNS TEXT AS $$ - SELECT option_value FROM pg_options_to_table(( - - SELECT fs.srvoptions - FROM pg_foreign_table ft - LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid - WHERE ft.ftrelid = reloid - - )) WHERE option_name='dbname'; -$$ LANGUAGE SQL; - - --- TODO: move to the right place --- Return the last updated time of a set of tables -CREATE OR REPLACE FUNCTION CDB_Last_Updated_Time(tables text[]) -RETURNS timestamptz AS $$ - WITH t AS ( - SELECT unnest(tables) AS schema_table_name - ), t_oid AS ( - SELECT (t.schema_table_name)::regclass::oid as reloid FROM t - ), t_updated_at AS ( - SELECT - (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) - ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) - END) AS updated_at - FROM t_oid - LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid - ) SELECT max(updated_at) FROM t_updated_at; -$$ LANGUAGE SQL; From 1c637f86896212357d6a4489307d0f8da9d52ca0 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 16:38:11 +0100 Subject: [PATCH 12/15] Small fixes: qualify cartodb function calls --- scripts-available/CDB_ForeignTable.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 6367512..b75fd46 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -136,7 +136,7 @@ LANGUAGE plpgsql; -- It is aware of foreign tables -- It assumes the local (schema_name, table_name) map to the remote ones with the same name CREATE OR REPLACE FUNCTION cartodb.CDB_QueryTables_Updated_At(query text) -RETURNS TABLE(db_name text, schema_name text, table_name text, updated_at timestamptz) +RETURNS TABLE(dbname text, schema_name text, table_name text, updated_at timestamptz) AS $$ WITH query_tables AS ( SELECT unnest(CDB_QueryTablesText(query)) schema_table_name @@ -146,7 +146,7 @@ AS $$ ), fqtn AS ( SELECT - (CASE WHEN c.relkind = 'f' THEN _cdb_dbname_of_foreign_table(query_tables_oid.reloid) + (CASE WHEN c.relkind = 'f' THEN cartodb._cdb_dbname_of_foreign_table(query_tables_oid.reloid) ELSE current_database() END)::text AS dbname, n.nspname::text schema_name, @@ -158,7 +158,7 @@ AS $$ WHERE c.oid = query_tables_oid.reloid ) SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, - (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) END) AS updated_at FROM fqtn; @@ -189,7 +189,7 @@ RETURNS timestamptz AS $$ SELECT (t.schema_table_name)::regclass::oid as reloid FROM t ), t_updated_at AS ( SELECT - (CASE WHEN relkind = 'f' THEN CDB_Get_Foreign_Updated_At(reloid) + (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) END) AS updated_at FROM t_oid From d59b826d37dde21b4ddd2f8989c8adfe34e72187 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 17:11:16 +0100 Subject: [PATCH 13/15] Fix silly bug: ordering of functions --- scripts-available/CDB_ForeignTable.sql | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index b75fd46..5729288 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -132,6 +132,19 @@ $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid) +RETURNS TEXT AS $$ + SELECT option_value FROM pg_options_to_table(( + + SELECT fs.srvoptions + FROM pg_foreign_table ft + LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid + WHERE ft.ftrelid = reloid + + )) WHERE option_name='dbname'; +$$ LANGUAGE SQL; + + -- Return a set of (dbname, schema_name, table_name, updated_at) -- It is aware of foreign tables -- It assumes the local (schema_name, table_name) map to the remote ones with the same name @@ -165,19 +178,6 @@ AS $$ $$ LANGUAGE SQL; -CREATE OR REPLACE FUNCTION cartodb._cdb_dbname_of_foreign_table(reloid oid) -RETURNS TEXT AS $$ - SELECT option_value FROM pg_options_to_table(( - - SELECT fs.srvoptions - FROM pg_foreign_table ft - LEFT JOIN pg_foreign_server fs ON ft.ftserver = fs.oid - WHERE ft.ftrelid = reloid - - )) WHERE option_name='dbname'; -$$ LANGUAGE SQL; - - -- Return the last updated time of a set of tables -- It is aware of foreign tables -- It assumes the local (schema_name, table_name) map to the remote ones with the same name From cd4ad29e39c8e437cb3875aaa2995a5d808c2869 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 18:49:17 +0100 Subject: [PATCH 14/15] Remove schema when selecting from CDB_TableMetadata to ease testing. When creating CDB_TableMetadata it is always put in cartodb when done from the extension, but for tests it is done in public. --- scripts-available/CDB_ForeignTable.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 5729288..a7ef87e 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -172,7 +172,7 @@ AS $$ ) SELECT fqtn.dbname, fqtn.schema_name, fqtn.table_name, (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) - ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) END) AS updated_at FROM fqtn; $$ LANGUAGE SQL; @@ -190,7 +190,7 @@ RETURNS timestamptz AS $$ ), t_updated_at AS ( SELECT (CASE WHEN relkind = 'f' THEN cartodb.CDB_Get_Foreign_Updated_At(reloid) - ELSE (SELECT md.updated_at FROM cartodb.CDB_TableMetadata md WHERE md.tabname = reloid) + ELSE (SELECT md.updated_at FROM CDB_TableMetadata md WHERE md.tabname = reloid) END) AS updated_at FROM t_oid LEFT JOIN pg_catalog.pg_class c ON c.oid = reloid From 2bce771488a47d3d689fcd422ee008533da7d187 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Feb 2016 18:50:50 +0100 Subject: [PATCH 15/15] Add some tests Tests for: * CDB_QueryTables_Updated_At * CDB_Last_Updated_Time --- test/extension/test.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/extension/test.sh b/test/extension/test.sh index 949cbe4..cf9f830 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -487,6 +487,19 @@ test_fdw|foo|test_fdw" sql postgres "SELECT a from test_fdw.foo LIMIT 1;" should 42 + # Check function CDB_QueryTables_Updated_At + sql postgres 'CREATE TABLE local (b int);' + sql postgres 'INSERT INTO local (b) VALUES (43);' + sql postgres "SELECT cdb_tablemetadatatouch('public.local'::regclass);" + local query='$query$ SELECT * FROM test_fdw.foo, local $query$::text' + sql postgres "SELECT dbname, schema_name, table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) ORDER BY dbname;" should 'fdw_target|test_fdw|foo +test_extension|public|local' + sql postgres "SELECT table_name FROM cartodb.CDB_QueryTables_Updated_At(${query}) order by updated_at;" should 'foo +local' + + # Check function CDB_Last_Updated_Time + sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) < now()" should 't' + sql postgres "SELECT cartodb.CDB_Last_Updated_Time('{test_fdw.foo,public.local}'::text[]) > (now() - interval '1 minute')" should 't' DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;'