From 5f154a585910f4de5df08c1287d0fd5e02287f78 Mon Sep 17 00:00:00 2001 From: Gonzalo Riestra Date: Fri, 8 Mar 2019 08:28:06 +0100 Subject: [PATCH] use TIS instead of Redis --- scripts-available/CDB_GhostTables.sql | 32 ++++++++++++++----------- test/CDB_GhostTables.sql | 34 +++++++++++++-------------- test/CDB_GhostTables_expect | 6 ++--- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index caf0258..ded2883 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -1,27 +1,28 @@ -- Table to store the transaction id from DDL events to avoid multiple executions -CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY); +CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text); -- Enqueues a job to run Ghost tables linking process for the provided user_id -CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(user_id text) +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, ddl_tag text) RETURNS void AS $$ - if not user_id: + if not username: return client = GD.get('redis', None) retry = 3 error = '' - redis_host = '127.0.0.1' - redis_port = 6379 - redis_timeout = 5 + # TODO: read TIS config from cdb_conf + tis_host = '127.0.0.1' + tis_port = 6379 + tis_timeout = 5 while True: if not client: try: import redis - client = GD['redis'] = redis.Redis(host=redis_host, port=redis_port, socket_timeout=redis_timeout) + client = GD['redis'] = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) except Exception as err: error = "client_error - %s" % str(err) # NOTE: no retries on connection error @@ -29,8 +30,7 @@ AS $$ break try: - job = '{{"class":"Resque::UserDBJobs::UserDBMaintenance::LinkGhostTables","args":["{}"]}}'.format(user_id) - client.rpush("resque:queue:user_dbs", job) + # client.execute_command('DBSCH', db_name, username, ddl_tag) break except Exception as err: error = "request_error - %s" % str(err) @@ -46,12 +46,16 @@ CREATE OR REPLACE FUNCTION CDB_LinkGhostTables() RETURNS void AS $$ DECLARE - user_id TEXT; + username TEXT; + db_name TEXT; + ddl_tag TEXT; BEGIN - EXECUTE 'SELECT (regexp_match(session_user, ''[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''))[1];' INTO user_id; - PERFORM _CDB_LinkGhostTables(user_id); + EXECUTE 'SELECT CDB_Username();' INTO username; + EXECUTE 'SELECT current_database();' INTO db_name; + EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag; + PERFORM _CDB_LinkGhostTables(username, db_name, ddl_tag); DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); - RAISE NOTICE '_CDB_LinkGhostTables(%) called', user_id; + RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, ddl_tag=%', username, ddl_tag; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; @@ -78,7 +82,7 @@ CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction() RETURNS event_trigger AS $$ BEGIN - INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current()) ON CONFLICT (txid) DO NOTHING; + INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING; END; $$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index 64285b7..c8c7148 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -1,20 +1,20 @@ -- Create user and enable Ghost tables trigger \set QUIET on -CREATE ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db" LOGIN; -CREATE SCHEMA fulano; -GRANT ALL ON SCHEMA fulano TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT USAGE ON SCHEMA cartodb TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT SELECT ON cartodb.cdb_ddl_execution TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +CREATE ROLE "fulano" LOGIN; +GRANT ALL ON SCHEMA cartodb TO "fulano"; +GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables() TO "fulano"; SELECT CDB_EnableGhostTablesTrigger(); -SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); +SET SESSION AUTHORIZATION "fulano"; \set QUIET off SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called BEGIN; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -CREATE TABLE fulano.tmp(id INT); +CREATE TABLE tmp(id INT); SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1 END; -- _CDB_LinkGhostTables called @@ -22,23 +22,21 @@ END; -- _CDB_LinkGhostTables called \set QUIET on SET SESSION AUTHORIZATION postgres; SELECT CDB_DisableGhostTablesTrigger(); -DROP TABLE fulano.tmp; -SET SESSION AUTHORIZATION "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +SET SESSION AUTHORIZATION "fulano"; \set QUIET off BEGIN; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 -CREATE TABLE fulano.tmp(id INT); +DROP TABLE tmp; SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 END; -- _CDB_LinkGhostTables not called --- Clean test stuff +-- Clean up \set QUIET on SET SESSION AUTHORIZATION postgres; -DROP TABLE fulano.tmp; -REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE USAGE ON SCHEMA cartodb FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -REVOKE ALL ON SCHEMA fulano FROM "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; -DROP ROLE "cartodb_user_b621f453-75ee-438f-acc9-8c5303f3d658_db"; +REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables() FROM "fulano"; +REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano"; +REVOKE SELECT ON cartodb.cdb_ddl_execution FROM "fulano"; +REVOKE ALL ON SCHEMA cartodb FROM "fulano"; +DROP ROLE "fulano"; \set QUIET off diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 8d79b24..33b0945 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -1,15 +1,15 @@ -NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag= BEGIN 0 CREATE TABLE 1 -NOTICE: _CDB_LinkGhostTables(b621f453-75ee-438f-acc9-8c5303f3d658) called +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, ddl_tag=CREATE TABLE COMMIT BEGIN 0 -CREATE TABLE +DROP TABLE 0 COMMIT