Merge pull request #349 from CartoDB/ghost_tables

Ghost tables functions

.travis.yml

@@ -26,7 +26,7 @@ before_install:
   - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
   - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust
   - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
-
+  - sudo pip install redis==2.4.9
 script:
   - make
   - sudo make install

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.24.1
+EXTVERSION = 0.26.0
 
 SED = sed
 AWK = awk
@@ -93,6 +93,8 @@ UPGRADABLE = \
   0.23.2 \
   0.24.0 \
   0.24.1 \
+  0.25.0 \
+  0.26.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,10 @@
+0.26.0 (2019-03-11)
+* Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`.
+* Add Ghost tables functions to install triggers and enqueue the linking process
+
+0.25.0 (2019-02-22)
+* Add `CDB_Username` to get the cartodb username from the current PostgreSQL user
+
 0.24.1 (2019-01-02)
 * Drop functions removed in 0.12 (#341)
 * Travis: Test with PostgreSQL 9.5, 10 and 11.

scripts-available/CDB_GhostTables.sql

@@ -0,0 +1,120 @@
+-- Enqueues a job to run Ghost tables linking process for the provided username
+CREATE OR REPLACE FUNCTION _CDB_LinkGhostTables(username text, db_name text, event_name text) 
+RETURNS void
+AS $$
+  if not username:
+    return
+
+  if 'json' not in GD:
+    import json
+    GD['json'] = json
+  else:
+    json = GD['json']    
+
+  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
+  tis_config_dict = json.loads(tis_config) if tis_config else {}
+  tis_host = tis_config_dict.get('host', '127.0.0.1')
+  tis_port = tis_config_dict.get('port', 3142)
+  tis_timeout = tis_config_dict.get('timeout', 5)
+  tis_retry = tis_config_dict.get('retry', 5)
+      
+  client = GD.get('invalidation', None)
+
+  while True:
+
+    if not client:
+        try:
+          import redis
+          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
+          GD['invalidation'] = client
+        except Exception as err:
+          error = "client_error - %s" % str(err)
+          # NOTE: no retries on connection error
+          plpy.warning('Invalidation Service connection error: ' +  str(err))
+          break
+
+    try:
+      client.execute_command('DBSCH', db_name, username, event_name)
+      break
+    except Exception as err:
+      error = "request_error - %s" % str(err)
+      client = GD['invalidation'] = None # force reconnect
+      if not tis_retry:
+        plpy.warning('Invalidation Service error: ' +  str(err))
+        break
+      tis_retry -= 1 # try reconnecting
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+
+-- Enqueues a job to run Ghost tables linking process for the current user
+CREATE OR REPLACE FUNCTION CDB_LinkGhostTables(event_name text DEFAULT 'USER')
+RETURNS void
+AS $$
+  DECLARE
+    username TEXT;
+    db_name TEXT;
+  BEGIN
+    EXECUTE 'SELECT CDB_Username();' INTO username;
+    EXECUTE 'SELECT current_database();' INTO db_name;
+
+    PERFORM _CDB_LinkGhostTables(username, db_name, event_name);
+    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Trigger function to call CDB_LinkGhostTables()
+CREATE OR REPLACE FUNCTION _CDB_LinkGhostTablesTrigger()
+RETURNS trigger
+AS $$
+  DECLARE
+    ddl_tag TEXT;
+  BEGIN
+    EXECUTE 'SELECT tag FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();' INTO ddl_tag;
+    DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current();
+    PERFORM CDB_LinkGhostTables(ddl_tag);
+    RETURN NULL;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Event trigger to save the current transaction in cartodb.cdb_ddl_execution
+CREATE OR REPLACE FUNCTION CDB_SaveDDLTransaction()
+RETURNS event_trigger
+AS $$
+  BEGIN
+    INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Creates the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION CDB_EnableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+
+    -- Table to store the transaction id from DDL events to avoid multiple executions
+    CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text);
+
+    CREATE CONSTRAINT TRIGGER check_ddl_update
+    AFTER INSERT ON cartodb.cdb_ddl_execution
+    INITIALLY DEFERRED
+    FOR EACH ROW
+    EXECUTE PROCEDURE _CDB_LinkGhostTablesTrigger();
+
+    CREATE EVENT TRIGGER link_ghost_tables
+    ON ddl_command_end
+    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW')
+    EXECUTE PROCEDURE CDB_SaveDDLTransaction();
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+-- Drops the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION CDB_DisableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+    DROP TABLE IF EXISTS cartodb.cdb_ddl_execution;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_GreatCircle.sql

@@ -16,7 +16,7 @@ BEGIN
 
   IF ST_XMax(line) - ST_XMin(line) > 180 THEN
     line = ST_Difference(
-      ST_Shift_Longitude(line),
+      ST_ShiftLongitude(line),
 			ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001)
 		);
   END IF;

scripts-available/CDB_Username.sql

@@ -0,0 +1,6 @@
+-- Returns the cartodb username of the current PostgreSQL session
+CREATE OR REPLACE FUNCTION CDB_Username()
+RETURNS text
+AS $$
+  SELECT CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
+$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;

scripts-enabled/290-CDB_GhostTables.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_GhostTables.sql

scripts-enabled/920-CDB_Username.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Username.sql

test/CDB_GhostTables.sql

@@ -0,0 +1,43 @@
+-- Create user and enable Ghost tables trigger
+\set QUIET on
+SET client_min_messages TO error;
+SELECT CDB_EnableGhostTablesTrigger();
+CREATE ROLE "fulano" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "fulano";
+GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano";
+INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+INSERT INTO cdb_conf (key, value) VALUES ('invalidation_service', '{"host": "fake-tis-host"}');
+SET SESSION AUTHORIZATION "fulano";
+SET client_min_messages TO notice;
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called
+
+BEGIN;
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0
+CREATE TABLE tmp(id INT);
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1
+END; -- _CDB_LinkGhostTables called
+
+-- Disable Ghost tables trigger
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableGhostTablesTrigger();
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists
+DROP TABLE tmp; -- _CDB_LinkGhostTables not called
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano";
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano";
+REVOKE ALL ON SCHEMA cartodb FROM "fulano";
+DROP ROLE "fulano";
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service';
+\set QUIET off

test/CDB_GhostTables_expect

@@ -0,0 +1,15 @@
+
+WARNING:  Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+BEGIN
+cdb_ddl_execution
+0
+CREATE TABLE
+1
+WARNING:  Invalidation Service error: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+COMMIT
+
+
+DROP TABLE

test/CDB_Username.sql

@@ -0,0 +1,23 @@
+SELECT session_user; -- postgres
+SELECT CDB_Username(); -- (NULL)
+
+-- Add the role fulano with api_key and connect with it
+\set QUIET on
+CREATE ROLE fulano LOGIN;
+GRANT USAGE ON SCHEMA cartodb TO fulano;
+GRANT EXECUTE ON FUNCTION CDB_Username() TO fulano;
+INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+SET SESSION AUTHORIZATION fulano;
+\set QUIET off
+
+SELECT session_user; -- fulano
+SELECT CDB_Username(); -- fulanito
+
+-- Remove fulano
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE USAGE ON SCHEMA cartodb FROM fulano;
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano;
+DROP ROLE fulano;
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano';
+\set QUIET off

test/CDB_Username_expect

@@ -0,0 +1,4 @@
+postgres
+
+fulano
+fulanito