From 99dd7cefc7577b50ca9e0b698b652cc5e9a06481 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Wed, 6 Mar 2019 18:19:38 +0100 Subject: [PATCH 01/14] ghost tables functions using redis --- scripts-available/CDB_GhostTables.sql | 104 ++++++++++++++++++++++++ scripts-enabled/290-CDB_GhostTables.sql | 1 + test/CDB_GhostTables.sql | 44 ++++++++++ test/CDB_GhostTables_expect | 15 ++++ 4 files changed, 164 insertions(+) create mode 100644 scripts-available/CDB_GhostTables.sql create mode 120000 scripts-enabled/290-CDB_GhostTables.sql create mode 100644 test/CDB_GhostTables.sql create mode 100644 test/CDB_GhostTables_expect diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql new file mode 100644 index 0000000..caf0258 --- /dev/null +++ b/scripts-available/CDB_GhostTables.sql @@ -0,0 +1,104 @@ +-- Table to store the transaction id from DDL events to avoid multiple executions +CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY); + +-- Enqueues a job to run Ghost tables linking process for the provided user_id +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(user_id text) +RETURNS void +AS $$ + if not user_id: + return + + client = GD.get('redis', None) + + retry = 3 + error = '' + redis_host = '127.0.0.1' + redis_port = 6379 + redis_timeout = 5 + + while True: + + if not client: + try: + import redis + client = GD['redis'] = redis.Redis(host=redis_host, port=redis_port, socket_timeout=redis_timeout) + except Exception as err: + error = "client_error - %s" % str(err) + # NOTE: no retries on connection error + plpy.error('Ghost tables connection error: ' + str(err)) + break + + try: + job = '{{"class":"Resque::UserDBJobs::UserDBMaintenance::LinkGhostTables","args":["{}"]}}'.format(user_id) + client.rpush("resque:queue:user_dbs", job) + break + except Exception as err: + error = "request_error - %s" % str(err) + client = GD['redis'] = None # force reconnect + if not retry: + plpy.error('Ghost tables error: ' + str(err)) + break + retry -= 1 # try reconnecting +$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; + +-- Enqueues a job to run Ghost tables linking process for the current user +CREATE OR REPLACE FUNCTION CDB_LinkGhostTables() +RETURNS void +AS $$ + DECLARE + user_id TEXT; + BEGIN + EXECUTE 'SELECT (regexp_match(session_user, ''[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''))[1];' INTO user_id; + PERFORM _CDB_LinkGhostTables(user_id); + DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); + RAISE NOTICE '_CDB_LinkGhostTables(%) called', user_id; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Trigger function to call CDB_LinkGhostTables() +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTablesTrigger() +RETURNS trigger +AS $$ + BEGIN + PERFORM CDB_LinkGhostTables(); + RETURN NULL; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Trigger to call CDB_LinkGhostTables() when adding a row in cartodb.cdb_ddl_execution +DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; +CREATE CONSTRAINT TRIGGER check_ddl_update +AFTER INSERT ON cartodb.cdb_ddl_execution +INITIALLY DEFERRED +FOR EACH ROW +EXECUTE PROCEDURE _CDB_LinkGhostTablesTrigger(); + +-- Event trigger to save the current transaction in cartodb.cdb_ddl_execution +CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction() +RETURNS event_trigger +AS $$ + BEGIN + INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current()) ON CONFLICT (txid) DO NOTHING; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Creates the trigger on DDL events to link ghost tables +CREATE OR REPLACE FUNCTION CDB_EnableGhostTablesTrigger() +RETURNS void +AS $$ + BEGIN + CREATE EVENT TRIGGER link_ghost_tables + ON ddl_command_end + WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER') + EXECUTE PROCEDURE CDB_SaveDDLTransaction(); + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; + +-- Drops the trigger on DDL events to link ghost tables +CREATE OR REPLACE FUNCTION CDB_DisableGhostTablesTrigger() +RETURNS void +AS $$ + BEGIN + DROP EVENT TRIGGER link_ghost_tables; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; diff --git a/scripts-enabled/290-CDB_GhostTables.sql b/scripts-enabled/290-CDB_GhostTables.sql new file mode 120000 index 0000000..9be6429 --- /dev/null +++ b/scripts-enabled/290-CDB_GhostTables.sql @@ -0,0 +1 @@ +../scripts-available/CDB_GhostTables.sql \ No newline at end of file diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql new file mode 100644 index 0000000..64285b7 --- /dev/null +++ b/test/CDB_GhostTables.sql @@ -0,0 +1,44 @@ +-- Create user and enable Ghost tables trigger +\set QUIET on +CREATE ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db" LOGIN; +CREATE SCHEMA fulano; +GRANT ALL ON SCHEMA fulano TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +GRANT USAGE ON SCHEMA cartodb TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +GRANT SELECT ON cartodb.cdb_ddl_execution TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +SELECT CDB_EnableGhostTablesTrigger(); +SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +\set QUIET off + +SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called + +BEGIN; +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 +CREATE TABLE fulano.tmp(id INT); +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1 +END; -- _CDB_LinkGhostTables called + +-- Disable Ghost tables trigger +\set QUIET on +SET SESSION AUTHORIZATION postgres; +SELECT CDB_DisableGhostTablesTrigger(); +DROP TABLE fulano.tmp; +SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +\set QUIET off + +BEGIN; +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 +CREATE TABLE fulano.tmp(id INT); +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 +END; -- _CDB_LinkGhostTables not called + +-- Clean test stuff +\set QUIET on +SET SESSION AUTHORIZATION postgres; +DROP TABLE fulano.tmp; +REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +REVOKE USAGE ON SCHEMA cartodb FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +REVOKE ALL ON SCHEMA fulano FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +DROP ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +\set QUIET off diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect new file mode 100644 index 0000000..8d79b24 --- /dev/null +++ b/test/CDB_GhostTables_expect @@ -0,0 +1,15 @@ + +NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called + +BEGIN +0 +CREATE TABLE +1 +NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called +COMMIT + +BEGIN +0 +CREATE TABLE +0 +COMMIT From 5f154a585910f4de5df08c1287d0fd5e02287f78 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 08:28:06 +0100 Subject: [PATCH 02/14] use TIS instead of Redis --- scripts-available/CDB_GhostTables.sql | 32 ++++++++++++++----------- test/CDB_GhostTables.sql | 34 +++++++++++++-------------- test/CDB_GhostTables_expect | 6 ++--- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index caf0258..ded2883 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -1,27 +1,28 @@ -- Table to store the transaction id from DDL events to avoid multiple executions -CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY); +CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text); -- Enqueues a job to run Ghost tables linking process for the provided user_id -CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(user_id text) +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, ddl_tag text) RETURNS void AS $$ - if not user_id: + if not username: return client = GD.get('redis', None) retry = 3 error = '' - redis_host = '127.0.0.1' - redis_port = 6379 - redis_timeout = 5 + # TODO: read TIS config from cdb_conf + tis_host = '127.0.0.1' + tis_port = 6379 + tis_timeout = 5 while True: if not client: try: import redis - client = GD['redis'] = redis.Redis(host=redis_host, port=redis_port, socket_timeout=redis_timeout) + client = GD['redis'] = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) except Exception as err: error = "client_error - %s" % str(err) # NOTE: no retries on connection error @@ -29,8 +30,7 @@ AS $$ break try: - job = '{{"class":"Resque::UserDBJobs::UserDBMaintenance::LinkGhostTables","args":["{}"]}}'.format(user_id) - client.rpush("resque:queue:user_dbs", job) + # client.execute_command('DBSCH', db_name, username, ddl_tag) break except Exception as err: error = "request_error - %s" % str(err) @@ -46,12 +46,16 @@ CREATE OR REPLACE FUNCTION CDB_LinkGhostTables() RETURNS void AS $$ DECLARE - user_id TEXT; + username TEXT; + db_name TEXT; + ddl_tag TEXT; BEGIN - EXECUTE 'SELECT (regexp_match(session_user, ''[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''))[1];' INTO user_id; - PERFORM _CDB_LinkGhostTables(user_id); + EXECUTE 'SELECT CDB_Username();' INTO username; + EXECUTE 'SELECT current_database();' INTO db_name; + EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag; + PERFORM _CDB_LinkGhostTables(username, db_name, ddl_tag); DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); - RAISE NOTICE '_CDB_LinkGhostTables(%) called', user_id; + RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, ddl_tag=%', username, ddl_tag; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; @@ -78,7 +82,7 @@ CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction() RETURNS event_trigger AS $$ BEGIN - INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current()) ON CONFLICT (txid) DO NOTHING; + INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index 64285b7..c8c7148 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -1,20 +1,20 @@ -- Create user and enable Ghost tables trigger \set QUIET on -CREATE ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db" LOGIN; -CREATE SCHEMA fulano; -GRANT ALL ON SCHEMA fulano TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT USAGE ON SCHEMA cartodb TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT SELECT ON cartodb.cdb_ddl_execution TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +CREATE ROLE "fulano" LOGIN; +GRANT ALL ON SCHEMA cartodb TO "fulano"; +GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "fulano"; SELECT CDB_EnableGhostTablesTrigger(); -SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); +SET SESSION AUTHORIZATION "fulano"; \set QUIET off SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called BEGIN; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -CREATE TABLE fulano.tmp(id INT); +CREATE TABLE tmp(id INT); SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1 END; -- _CDB_LinkGhostTables called @@ -22,23 +22,21 @@ END; -- _CDB_LinkGhostTables called \set QUIET on SET SESSION AUTHORIZATION postgres; SELECT CDB_DisableGhostTablesTrigger(); -DROP TABLE fulano.tmp; -SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +SET SESSION AUTHORIZATION "fulano"; \set QUIET off BEGIN; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -CREATE TABLE fulano.tmp(id INT); +DROP TABLE tmp; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 END; -- _CDB_LinkGhostTables not called --- Clean test stuff +-- Clean up \set QUIET on SET SESSION AUTHORIZATION postgres; -DROP TABLE fulano.tmp; -REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE USAGE ON SCHEMA cartodb FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE ALL ON SCHEMA fulano FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -DROP ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "fulano"; +REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano"; +REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "fulano"; +REVOKE ALL ON SCHEMA cartodb FROM "fulano"; +DROP ROLE "fulano"; \set QUIET off diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 8d79b24..33b0945 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -1,15 +1,15 @@ -NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag= BEGIN 0 CREATE TABLE 1 -NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag=CREATE TABLE COMMIT BEGIN 0 -CREATE TABLE +DROP TABLE 0 COMMIT From ab6720ad32675dc4a3a2145d1b8e9b2c017585b2 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 08:28:24 +0100 Subject: [PATCH 03/14] add create/drop/alter view to the trigger --- scripts-available/CDB_GhostTables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index ded2883..2e2ddeb 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -93,7 +93,7 @@ AS $$ BEGIN CREATE EVENT TRIGGER link_ghost_tables ON ddl_command_end - WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER') + WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW') EXECUTE PROCEDURE CDB_SaveDDLTransaction(); END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; From e3138cd56a9081edb2934aa56a6173a6d072d43f Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 08:43:22 +0100 Subject: [PATCH 04/14] make enable/disable trigger idempotent --- scripts-available/CDB_GhostTables.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index 2e2ddeb..c1d916e 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -91,9 +91,10 @@ CREATE OR REPLACE FUNCTION CDB_EnableGhostTablesTrigger() RETURNS void AS $$ BEGIN + DROP EVENT TRIGGER IF EXISTS link_ghost_tables; CREATE EVENT TRIGGER link_ghost_tables ON ddl_command_end - WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW') + WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER') EXECUTE PROCEDURE CDB_SaveDDLTransaction(); END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; @@ -103,6 +104,6 @@ CREATE OR REPLACE FUNCTION CDB_DisableGhostTablesTrigger() RETURNS void AS $$ BEGIN - DROP EVENT TRIGGER link_ghost_tables; + DROP EVENT TRIGGER IF EXISTS link_ghost_tables; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; From 50e41179fce5dcc13620c935bc1e6030482aa5e0 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 09:18:46 +0100 Subject: [PATCH 05/14] fix test --- test/CDB_GhostTables.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index c8c7148..d80f095 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -1,5 +1,6 @@ -- Create user and enable Ghost tables trigger \set QUIET on +SET client_min_messages TO error; CREATE ROLE "fulano" LOGIN; GRANT ALL ON SCHEMA cartodb TO "fulano"; GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano"; @@ -8,6 +9,7 @@ GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "fulano"; SELECT CDB_EnableGhostTablesTrigger(); INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); SET SESSION AUTHORIZATION "fulano"; +SET client_min_messages TO notice; \set QUIET off SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called From a794fb3d31784ec4c6d1c633bc8317d3d35c3569 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 12:03:55 +0100 Subject: [PATCH 06/14] read tis config from cdb_conf --- scripts-available/CDB_GhostTables.sql | 33 ++++++++++++++++----------- test/CDB_GhostTables.sql | 2 ++ test/CDB_GhostTables_expect | 2 ++ test/CDB_Username.sql | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index c1d916e..c94b64f 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -8,37 +8,44 @@ AS $$ if not username: return - client = GD.get('redis', None) + if 'json' not in GD: + import json + GD['json'] = json + else: + json = GD['json'] - retry = 3 error = '' - # TODO: read TIS config from cdb_conf - tis_host = '127.0.0.1' - tis_port = 6379 - tis_timeout = 5 + tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf'] + tis_config_dict = json.loads(tis_config) if tis_config else {} + tis_host = tis_config_dict.get('host', '127.0.0.1') + tis_port = tis_config_dict.get('port', 3142) + tis_timeout = tis_config_dict.get('timeout', 5) + tis_retry = tis_config_dict.get('retry', 5) + + client = GD.get('invalidation', None) while True: if not client: try: import redis - client = GD['redis'] = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) + client = GD['invalidation'] = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) except Exception as err: error = "client_error - %s" % str(err) # NOTE: no retries on connection error - plpy.error('Ghost tables connection error: ' + str(err)) + plpy.warning('Invalidation Service connection error: ' + str(err)) break try: - # client.execute_command('DBSCH', db_name, username, ddl_tag) + client.execute_command('DBSCH', db_name, username, ddl_tag) break except Exception as err: error = "request_error - %s" % str(err) - client = GD['redis'] = None # force reconnect - if not retry: - plpy.error('Ghost tables error: ' + str(err)) + client = GD['invalidation'] = None # force reconnect + if not tis_retry: + plpy.warning('Invalidation Service error: ' + str(err)) break - retry -= 1 # try reconnecting + tis_retry -= 1 # try reconnecting $$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; -- Enqueues a job to run Ghost tables linking process for the current user diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index d80f095..e9aa8fa 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -8,6 +8,7 @@ GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano"; GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "fulano"; SELECT CDB_EnableGhostTablesTrigger(); INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); +INSERT INTO cdb_conf (key, value) VALUES ('invalidation_service', '{"host": "fake-tis-host"}'); SET SESSION AUTHORIZATION "fulano"; SET client_min_messages TO notice; \set QUIET off @@ -41,4 +42,5 @@ REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano"; REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "fulano"; REVOKE ALL ON SCHEMA cartodb FROM "fulano"; DROP ROLE "fulano"; +DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service'; \set QUIET off diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 33b0945..e6d6fb5 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -1,10 +1,12 @@ +WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag= BEGIN 0 CREATE TABLE 1 +WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag=CREATE TABLE COMMIT diff --git a/test/CDB_Username.sql b/test/CDB_Username.sql index ed344d6..55bd01c 100644 --- a/test/CDB_Username.sql +++ b/test/CDB_Username.sql @@ -19,4 +19,5 @@ SET SESSION AUTHORIZATION postgres; REVOKE USAGE ON SCHEMA cartodb FROM fulano; REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano; DROP ROLE fulano; +DELETE FROM cdb_conf WHERE key = 'api_keys_fulano'; \set QUIET off \ No newline at end of file From d1ee383d9bcf86ec906162f643f62fd7bdc50d2d Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 12:50:51 +0100 Subject: [PATCH 07/14] add redis module for python to travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ae454c6..812dcd5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - + - sudo pip install redis script: - make - sudo make install From cc1df0a708885a82b5207ef0336cc7e27c1fcfc1 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 13:01:56 +0100 Subject: [PATCH 08/14] use redis module version from carto --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 812dcd5..7f8e216 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - - sudo pip install redis + - sudo pip install redis==2.4.9 script: - make - sudo make install From 667f896cfb0c354803af34a38a8edf4b3976075d Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 13:21:11 +0100 Subject: [PATCH 09/14] add again views to the trigger --- scripts-available/CDB_GhostTables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index c94b64f..cdbad8f 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -101,7 +101,7 @@ AS $$ DROP EVENT TRIGGER IF EXISTS link_ghost_tables; CREATE EVENT TRIGGER link_ghost_tables ON ddl_command_end - WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER') + WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW') EXECUTE PROCEDURE CDB_SaveDDLTransaction(); END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; From c7bba14e9a91f0aa7b28f575adce5e142a84fcba Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Mon, 11 Mar 2019 09:20:05 +0100 Subject: [PATCH 10/14] simplify code --- scripts-available/CDB_GhostTables.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index cdbad8f..91d6a4d 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -29,7 +29,8 @@ AS $$ if not client: try: import redis - client = GD['invalidation'] = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) + client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) + GD['invalidation'] = client except Exception as err: error = "client_error - %s" % str(err) # NOTE: no retries on connection error From 9c6294d95b9f91e707498413d513e8e5c0180a9d Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Mon, 11 Mar 2019 11:28:45 +0100 Subject: [PATCH 11/14] move trigger drop/creation inside functions --- scripts-available/CDB_GhostTables.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index 91d6a4d..21b3d95 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -14,7 +14,6 @@ AS $$ else: json = GD['json'] - error = '' tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf'] tis_config_dict = json.loads(tis_config) if tis_config else {} tis_host = tis_config_dict.get('host', '127.0.0.1') @@ -77,14 +76,6 @@ AS $$ END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; --- Trigger to call CDB_LinkGhostTables() when adding a row in cartodb.cdb_ddl_execution -DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; -CREATE CONSTRAINT TRIGGER check_ddl_update -AFTER INSERT ON cartodb.cdb_ddl_execution -INITIALLY DEFERRED -FOR EACH ROW -EXECUTE PROCEDURE _CDB_LinkGhostTablesTrigger(); - -- Event trigger to save the current transaction in cartodb.cdb_ddl_execution CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction() RETURNS event_trigger @@ -100,6 +91,14 @@ RETURNS void AS $$ BEGIN DROP EVENT TRIGGER IF EXISTS link_ghost_tables; + DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; + + CREATE CONSTRAINT TRIGGER check_ddl_update + AFTER INSERT ON cartodb.cdb_ddl_execution + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE _CDB_LinkGhostTablesTrigger(); + CREATE EVENT TRIGGER link_ghost_tables ON ddl_command_end WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW') @@ -113,5 +112,6 @@ RETURNS void AS $$ BEGIN DROP EVENT TRIGGER IF EXISTS link_ghost_tables; + DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; From 1a271d977b4c482c21d5c2353442024255765822 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Mon, 11 Mar 2019 12:24:14 +0100 Subject: [PATCH 12/14] bump version --- Makefile | 3 ++- NEWS.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fa9ffb6..6a194e7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.25.0 +EXTVERSION = 0.26.0 SED = sed AWK = awk @@ -94,6 +94,7 @@ UPGRADABLE = \ 0.24.0 \ 0.24.1 \ 0.25.0 \ + 0.26.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index a777495..2d98860 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 0.26.0 (2019-XX-XX) * Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`. +* Add Ghost tables functions to install triggers and enqueue the linking process 0.25.0 (2019-02-22) * Add `CDB_Username` to get the cartodb username from the current PostgreSQL user From 65d51fd8bd382f31b275dc52013fd367a1801a3b Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Mon, 11 Mar 2019 14:02:45 +0100 Subject: [PATCH 13/14] move cdb_ddl_execution table creation/drop to functions --- scripts-available/CDB_GhostTables.sql | 29 +++++++++++++++------------ test/CDB_GhostTables.sql | 17 +++++++--------- test/CDB_GhostTables_expect | 10 ++++----- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index 21b3d95..2f9e40e 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -1,8 +1,5 @@ --- Table to store the transaction id from DDL events to avoid multiple executions -CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text); - --- Enqueues a job to run Ghost tables linking process for the provided user_id -CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, ddl_tag text) +-- Enqueues a job to run Ghost tables linking process for the provided username +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, event_name text) RETURNS void AS $$ if not username: @@ -37,7 +34,7 @@ AS $$ break try: - client.execute_command('DBSCH', db_name, username, ddl_tag) + client.execute_command('DBSCH', db_name, username, event_name) break except Exception as err: error = "request_error - %s" % str(err) @@ -49,20 +46,18 @@ AS $$ $$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; -- Enqueues a job to run Ghost tables linking process for the current user -CREATE OR REPLACE FUNCTION CDB_LinkGhostTables() +CREATE OR REPLACE FUNCTION CDB_LinkGhostTables(event_name text DEFAULT 'USER') RETURNS void AS $$ DECLARE username TEXT; db_name TEXT; - ddl_tag TEXT; BEGIN EXECUTE 'SELECT CDB_Username();' INTO username; EXECUTE 'SELECT current_database();' INTO db_name; - EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag; - PERFORM _CDB_LinkGhostTables(username, db_name, ddl_tag); - DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); - RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, ddl_tag=%', username, ddl_tag; + + PERFORM _CDB_LinkGhostTables(username, db_name, event_name); + RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; @@ -70,8 +65,12 @@ $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; CREATE OR REPLACE FUNCTION _CDB_LinkGhostTablesTrigger() RETURNS trigger AS $$ + DECLARE + ddl_tag TEXT; BEGIN - PERFORM CDB_LinkGhostTables(); + EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag; + DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); + PERFORM CDB_LinkGhostTables(ddl_tag); RETURN NULL; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; @@ -93,6 +92,9 @@ AS $$ DROP EVENT TRIGGER IF EXISTS link_ghost_tables; DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; + -- Table to store the transaction id from DDL events to avoid multiple executions + CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text); + CREATE CONSTRAINT TRIGGER check_ddl_update AFTER INSERT ON cartodb.cdb_ddl_execution INITIALLY DEFERRED @@ -113,5 +115,6 @@ AS $$ BEGIN DROP EVENT TRIGGER IF EXISTS link_ghost_tables; DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; + DROP TABLE IF EXISTS cartodb.cdb_ddl_execution; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index e9aa8fa..33fae5b 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -1,12 +1,12 @@ -- Create user and enable Ghost tables trigger \set QUIET on SET client_min_messages TO error; +SELECT CDB_EnableGhostTablesTrigger(); CREATE ROLE "fulano" LOGIN; GRANT ALL ON SCHEMA cartodb TO "fulano"; GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano"; GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano"; -GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "fulano"; -SELECT CDB_EnableGhostTablesTrigger(); +GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano"; INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); INSERT INTO cdb_conf (key, value) VALUES ('invalidation_service', '{"host": "fake-tis-host"}'); SET SESSION AUTHORIZATION "fulano"; @@ -16,6 +16,7 @@ SET client_min_messages TO notice; SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called BEGIN; +SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 CREATE TABLE tmp(id INT); SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1 @@ -28,18 +29,14 @@ SELECT CDB_DisableGhostTablesTrigger(); SET SESSION AUTHORIZATION "fulano"; \set QUIET off -BEGIN; -SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -DROP TABLE tmp; -SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -END; -- _CDB_LinkGhostTables not called +SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists +DROP TABLE tmp; -- _CDB_LinkGhostTables not called --- Clean up +-- Cleanup \set QUIET on SET SESSION AUTHORIZATION postgres; -REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "fulano"; +REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano"; REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano"; -REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "fulano"; REVOKE ALL ON SCHEMA cartodb FROM "fulano"; DROP ROLE "fulano"; DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service'; diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index e6d6fb5..3e46518 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -1,17 +1,15 @@ WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. -NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag= +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER BEGIN +cdb_ddl_execution 0 CREATE TABLE 1 WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. -NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag=CREATE TABLE +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE COMMIT -BEGIN -0 + DROP TABLE -0 -COMMIT From b1202011f6f6f3efc28a7eb384d63443e1d75a9f Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Mon, 11 Mar 2019 15:28:48 +0100 Subject: [PATCH 14/14] update news --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2d98860..7bba82a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -0.26.0 (2019-XX-XX) +0.26.0 (2019-03-11) * Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`. * Add Ghost tables functions to install triggers and enqueue the linking process