From c06d24aa191956fdcde1238e31faaa20ab68becc Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 28 Jun 2019 14:52:37 +0200 Subject: [PATCH 1/4] Make CDB_Get_Foreign_Updated_At robust to missing CDB_TableMetadata This may happen with non-carto DB's, when checking the updated_at times and not finding the corresponding remote.cdb_tablemetadata imported from the foreign non-carto DB. Instead of failing, return a NOW() timestampt, so that caching logic just assumes there may have been changes. This makes it work today, and leaves open the possibility of adding the required carto metadata for homogeneous caching in the future. --- scripts-available/CDB_ForeignTable.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 98d3c9b..0dc8bdf 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -125,7 +125,15 @@ BEGIN -- We assume that the remote cdb_tablemetadata is called cdb_tablemetadata and is on the same schema as the queried table. SELECT nspname FROM pg_class c, pg_namespace n WHERE c.oid=foreign_table AND c.relnamespace = n.oid INTO fdw_schema_name; - EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time; + BEGIN + EXECUTE FORMAT('SELECT updated_at FROM %I.cdb_tablemetadata WHERE tabname=%L ORDER BY updated_at DESC LIMIT 1', fdw_schema_name, remote_table_name) INTO time; + EXCEPTION + WHEN undefined_table THEN + -- If you add a GET STACKED DIAGNOSTICS text_var = RETURNED_SQLSTATE + -- you get a code 42P01 which corresponds to undefined_table + RAISE NOTICE 'CDB_Get_Foreign_Updated_At: could not find %.cdb_tablemetadata while checking % updated_at, returning NOW() timestamp', fdw_schema_name, foreign_table; + time := NOW(); + END; RETURN time; END $$ From b0b4a922408e5a9e2d3db4aea6b9b12e76109bff Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 28 Jun 2019 15:55:42 +0200 Subject: [PATCH 2/4] Add a test case for missing foreign CDB_Tablementadata --- test/extension/test.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/extension/test.sh b/test/extension/test.sh index 54c2c50..62704ee 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -586,6 +586,12 @@ test_extension|public|"local-table-with-dashes"' sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) < now()" should 't' sql postgres "SELECT cartodb.CDB_Last_Updated_Time(ARRAY['test_extension.public.\"local-table-with-dashes\"']::text[]) > (now() - interval '1 minute')" should 't' + # Check CDB_Get_Foreign_Updated_At is robust to unimported CDB_TableMetadata + sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;" + sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') < (now() + interval '1 minute')" should 't' + sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') > (now() - interval '1 minute')" should 't' + + # Teardown DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo FROM fdw_user;' DATABASE=fdw_target sql postgres 'REVOKE SELECT ON test_fdw.foo2 FROM fdw_user;' From 446f4113d9c3d0f99d043920ac49d4bc4c5f53e6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 3 Jul 2019 16:19:46 +0200 Subject: [PATCH 3/4] Return NULL instead of NOW() The absence of foreign CDB_TableMetadata actually means that we cannot really tell when a remote table was modified. Therefore we're using NULL with the meaning of "I don't know when it was last modified". To be taken in other caching layers, to adjust headers accordingly. --- scripts-available/CDB_ForeignTable.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts-available/CDB_ForeignTable.sql b/scripts-available/CDB_ForeignTable.sql index 0dc8bdf..f60b043 100644 --- a/scripts-available/CDB_ForeignTable.sql +++ b/scripts-available/CDB_ForeignTable.sql @@ -131,8 +131,7 @@ BEGIN WHEN undefined_table THEN -- If you add a GET STACKED DIAGNOSTICS text_var = RETURNED_SQLSTATE -- you get a code 42P01 which corresponds to undefined_table - RAISE NOTICE 'CDB_Get_Foreign_Updated_At: could not find %.cdb_tablemetadata while checking % updated_at, returning NOW() timestamp', fdw_schema_name, foreign_table; - time := NOW(); + RAISE NOTICE 'CDB_Get_Foreign_Updated_At: could not find %.cdb_tablemetadata while checking % updated_at, returning NULL timestamp', fdw_schema_name, foreign_table; END; RETURN time; END From d378ca6fe03cb98e1861689eec994826bef420a4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 3 Jul 2019 16:25:16 +0200 Subject: [PATCH 4/4] Test for missing foreign CDB_TableMetadata --- test/extension/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/extension/test.sh b/test/extension/test.sh index 62704ee..1d9e7ee 100755 --- a/test/extension/test.sh +++ b/test/extension/test.sh @@ -588,8 +588,7 @@ test_extension|public|"local-table-with-dashes"' # Check CDB_Get_Foreign_Updated_At is robust to unimported CDB_TableMetadata sql postgres "DROP FOREIGN TABLE IF EXISTS test_fdw.cdb_tablemetadata;" - sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') < (now() + interval '1 minute')" should 't' - sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') > (now() - interval '1 minute')" should 't' + sql postgres "SELECT cartodb.CDB_Get_Foreign_Updated_At('test_fdw.foo') IS NULL" should 't' # Teardown DATABASE=fdw_target sql postgres 'REVOKE USAGE ON SCHEMA test_fdw FROM fdw_user;'