diff --git a/.travis.yml b/.travis.yml index ae454c6..7f8e216 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - + - sudo pip install redis==2.4.9 script: - make - sudo make install diff --git a/Makefile b/Makefile index fa9ffb6..6a194e7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.25.0 +EXTVERSION = 0.26.0 SED = sed AWK = awk @@ -94,6 +94,7 @@ UPGRADABLE = \ 0.24.0 \ 0.24.1 \ 0.25.0 \ + 0.26.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index a777495..7bba82a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ -0.26.0 (2019-XX-XX) +0.26.0 (2019-03-11) * Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`. +* Add Ghost tables functions to install triggers and enqueue the linking process 0.25.0 (2019-02-22) * Add `CDB_Username` to get the cartodb username from the current PostgreSQL user diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql new file mode 100644 index 0000000..2f9e40e --- /dev/null +++ b/scripts-available/CDB_GhostTables.sql @@ -0,0 +1,120 @@ +-- Enqueues a job to run Ghost tables linking process for the provided username +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, event_name text) +RETURNS void +AS $$ + if not username: + return + + if 'json' not in GD: + import json + GD['json'] = json + else: + json = GD['json'] + + tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf'] + tis_config_dict = json.loads(tis_config) if tis_config else {} + tis_host = tis_config_dict.get('host', '127.0.0.1') + tis_port = tis_config_dict.get('port', 3142) + tis_timeout = tis_config_dict.get('timeout', 5) + tis_retry = tis_config_dict.get('retry', 5) + + client = GD.get('invalidation', None) + + while True: + + if not client: + try: + import redis + client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) + GD['invalidation'] = client + except Exception as err: + error = "client_error - %s" % str(err) + # NOTE: no retries on connection error + plpy.warning('Invalidation Service connection error: ' + str(err)) + break + + try: + client.execute_command('DBSCH', db_name, username, event_name) + break + except Exception as err: + error = "request_error - %s" % str(err) + client = GD['invalidation'] = None # force reconnect + if not tis_retry: + plpy.warning('Invalidation Service error: ' + str(err)) + break + tis_retry -= 1 # try reconnecting +$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; + +-- Enqueues a job to run Ghost tables linking process for the current user +CREATE OR REPLACE FUNCTION CDB_LinkGhostTables(event_name text DEFAULT 'USER') +RETURNS void +AS $$ + DECLARE + username TEXT; + db_name TEXT; + BEGIN + EXECUTE 'SELECT CDB_Username();' INTO username; + EXECUTE 'SELECT current_database();' INTO db_name; + + PERFORM _CDB_LinkGhostTables(username, db_name, event_name); + RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Trigger function to call CDB_LinkGhostTables() +CREATE OR REPLACE FUNCTION _CDB_LinkGhostTablesTrigger() +RETURNS trigger +AS $$ + DECLARE + ddl_tag TEXT; + BEGIN + EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag; + DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current(); + PERFORM CDB_LinkGhostTables(ddl_tag); + RETURN NULL; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Event trigger to save the current transaction in cartodb.cdb_ddl_execution +CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction() +RETURNS event_trigger +AS $$ + BEGIN + INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER; + +-- Creates the trigger on DDL events to link ghost tables +CREATE OR REPLACE FUNCTION CDB_EnableGhostTablesTrigger() +RETURNS void +AS $$ + BEGIN + DROP EVENT TRIGGER IF EXISTS link_ghost_tables; + DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; + + -- Table to store the transaction id from DDL events to avoid multiple executions + CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text); + + CREATE CONSTRAINT TRIGGER check_ddl_update + AFTER INSERT ON cartodb.cdb_ddl_execution + INITIALLY DEFERRED + FOR EACH ROW + EXECUTE PROCEDURE _CDB_LinkGhostTablesTrigger(); + + CREATE EVENT TRIGGER link_ghost_tables + ON ddl_command_end + WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW') + EXECUTE PROCEDURE CDB_SaveDDLTransaction(); + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; + +-- Drops the trigger on DDL events to link ghost tables +CREATE OR REPLACE FUNCTION CDB_DisableGhostTablesTrigger() +RETURNS void +AS $$ + BEGIN + DROP EVENT TRIGGER IF EXISTS link_ghost_tables; + DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution; + DROP TABLE IF EXISTS cartodb.cdb_ddl_execution; + END; +$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE; diff --git a/scripts-enabled/290-CDB_GhostTables.sql b/scripts-enabled/290-CDB_GhostTables.sql new file mode 120000 index 0000000..9be6429 --- /dev/null +++ b/scripts-enabled/290-CDB_GhostTables.sql @@ -0,0 +1 @@ +../scripts-available/CDB_GhostTables.sql \ No newline at end of file diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql new file mode 100644 index 0000000..33fae5b --- /dev/null +++ b/test/CDB_GhostTables.sql @@ -0,0 +1,43 @@ +-- Create user and enable Ghost tables trigger +\set QUIET on +SET client_min_messages TO error; +SELECT CDB_EnableGhostTablesTrigger(); +CREATE ROLE "fulano" LOGIN; +GRANT ALL ON SCHEMA cartodb TO "fulano"; +GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano"; +GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano"; +INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}'); +INSERT INTO cdb_conf (key, value) VALUES ('invalidation_service', '{"host": "fake-tis-host"}'); +SET SESSION AUTHORIZATION "fulano"; +SET client_min_messages TO notice; +\set QUIET off + +SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called + +BEGIN; +SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0 +CREATE TABLE tmp(id INT); +SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1 +END; -- _CDB_LinkGhostTables called + +-- Disable Ghost tables trigger +\set QUIET on +SET SESSION AUTHORIZATION postgres; +SELECT CDB_DisableGhostTablesTrigger(); +SET SESSION AUTHORIZATION "fulano"; +\set QUIET off + +SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists +DROP TABLE tmp; -- _CDB_LinkGhostTables not called + +-- Cleanup +\set QUIET on +SET SESSION AUTHORIZATION postgres; +REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano"; +REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano"; +REVOKE ALL ON SCHEMA cartodb FROM "fulano"; +DROP ROLE "fulano"; +DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service'; +\set QUIET off diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect new file mode 100644 index 0000000..3e46518 --- /dev/null +++ b/test/CDB_GhostTables_expect @@ -0,0 +1,15 @@ + +WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER + +BEGIN +cdb_ddl_execution +0 +CREATE TABLE +1 +WARNING: Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known. +NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE +COMMIT + + +DROP TABLE diff --git a/test/CDB_Username.sql b/test/CDB_Username.sql index ed344d6..55bd01c 100644 --- a/test/CDB_Username.sql +++ b/test/CDB_Username.sql @@ -19,4 +19,5 @@ SET SESSION AUTHORIZATION postgres; REVOKE USAGE ON SCHEMA cartodb FROM fulano; REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano; DROP ROLE fulano; +DELETE FROM cdb_conf WHERE key = 'api_keys_fulano'; \set QUIET off \ No newline at end of file