release 0.26.1

Remove default TIS config in Ghost Table function

.travis.yml

@@ -1,26 +1,38 @@
+dist: xenial
 language: c
-dist: trusty
 sudo: required
 
 env:
-  - PGUSER=postgres PGDATABASE=postgres
+  global:
+    - PGUSER=postgres
+    - PGDATABASE=postgres
+    - PGOPTIONS='-c client_min_messages=NOTICE'
+    - PGPORT=5432
+    - POSTGIS_VERSION="2.5"
+
+  matrix:
+    - POSTGRESQL_VERSION="9.5"
+    - POSTGRESQL_VERSION="10"
+    - POSTGRESQL_VERSION="11"
 
-env:
-    - POSTGRESQL_VERSION="9.5", POSTGIS_VERSION="2.4"
-    - POSTGRESQL_VERSION="10", POSTGIS_VERSION="2.4"
 
 before_install:
-  - export POSTGIS_PACKAGE_VERSION=$(apt list postgis -a 2>/dev/null | grep $POSTGIS_VERSION | awk '{ print $2 }')
   - sudo service postgresql stop;
-  - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgis=$POSTGIS_PACKAGE_VERSION postgresql-$POSTGRESQL_VERSION-postgis-scripts postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-plpython-$POSTGRESQL_VERSION
-  - echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust"  | sudo tee /etc/postgresql/$POSTGRESQL_VERSION/main/pg_hba.conf
-  - export PGPORT=`grep ^port /etc/postgresql/$POSTGRESQL_VERSION/main/postgresql.conf | awk '{print $3}'`
-  - export PGUSER=postgres
-  - export PGDATABASE=postgres
-  - sudo service postgresql restart $POSTGRESQL_VERSION;
-
+  - sudo apt-get remove postgresql* -y
+  - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common
+  - if [[ $POSTGRESQL_VERSION == '9.5' ]]; then sudo apt-get install -y postgresql-contrib-9.5; fi;
+  - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION
+  - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main
+  - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION
+  - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust
+  - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe
+  - sudo pip install redis==2.4.9
 script:
   - make
   - sudo make install
-  - PGOPTIONS='-c client_min_messages=NOTICE' PGUSER=postgres make installcheck ||
-    { cat regression.diffs; false; }
+  - make installcheck
+
+after_failure:
+  - pg_lsclusters
+  - cat regression.out
+  - cat regression.diffs

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.24.0
+EXTVERSION = 0.26.1
 
 SED = sed
 AWK = awk
@@ -92,6 +92,10 @@ UPGRADABLE = \
   0.23.1 \
   0.23.2 \
   0.24.0 \
+  0.24.1 \
+  0.25.0 \
+  0.26.0 \
+  0.26.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,17 @@
+0.26.1 (2019-03-19)
+* Remove default TIS values from Ghost tables functions
+
+0.26.0 (2019-03-11)
+* Use `ST_ShiftLongitude` instead of `ST_Shift_Longitude`.
+* Add Ghost tables functions to install triggers and enqueue the linking process
+
+0.25.0 (2019-02-22)
+* Add `CDB_Username` to get the cartodb username from the current PostgreSQL user
+
+0.24.1 (2019-01-02)
+* Drop functions removed in 0.12 (#341)
+* Travis: Test with PostgreSQL 9.5, 10 and 11.
+
 0.24.0 (2018-09-13)
 * Travis: Test with PostgreSQL 9.5 and 10.
 * _cdb_estimated_extent: Fix bug with ST_EstimatedExtent interaction.

scripts-available/CDB_DDLTriggers.sql

@@ -0,0 +1,14 @@
+--
+-- Legacy file
+-- Introduced again to make sure that updates do not leave dangling functions
+--
+
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_create_table();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_drop_table();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_alter_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_drop_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_handle_add_column();
+DROP FUNCTION IF EXISTS cartodb.cdb_disable_ddl_hooks();
+DROP FUNCTION IF EXISTS cartodb.cdb_enable_ddl_hooks();
+
+

scripts-available/CDB_GhostTables.sql

@@ -0,0 +1,123 @@
+-- Enqueues a job to run Ghost tables linking process for the provided username
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text) 
+RETURNS void
+AS $$
+  if not username:
+    return
+
+  if 'json' not in GD:
+    import json
+    GD['json'] = json
+  else:
+    json = GD['json']    
+
+  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
+  if not tis_config:
+    plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
+    return
+
+  tis_config_dict = json.loads(tis_config)
+  tis_host = tis_config_dict.get('host')
+  tis_port = tis_config_dict.get('port')
+  tis_timeout = tis_config_dict.get('timeout', 5)
+  tis_retry = tis_config_dict.get('retry', 5)
+      
+  client = GD.get('invalidation', None)
+
+  while True:
+
+    if not client:
+        try:
+          import redis
+          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
+          GD['invalidation'] = client
+        except Exception as err:
+          error = "client_error - %s" % str(err)
+          # NOTE: no retries on connection error
+          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' +  str(err))
+          break
+
+    try:
+      client.execute_command('DBSCH', db_name, username, event_name)
+      break
+    except Exception as err:
+      error = "request_error - %s" % str(err)
+      client = GD['invalidation'] = None # force reconnect
+      if not tis_retry:
+        plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' +  str(err))
+        break
+      tis_retry -= 1 # try reconnecting
+$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+
+-- Enqueues a job to run Ghost tables linking process for the current user
+CREATE OR REPLACE FUNCTION cartodb.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
+RETURNS void
+AS $$
+  DECLARE
+    username TEXT;
+    db_name TEXT;
+  BEGIN
+    EXECUTE 'SELECT cartodb.CDB_Username();' INTO username;
+    EXECUTE 'SELECT current_database();' INTO db_name;
+
+    PERFORM cartodb._CDB_LinkGhostTables(username, db_name, event_name);
+    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Trigger function to call CDB_LinkGhostTables()
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTablesTrigger()
+RETURNS trigger
+AS $$
+  DECLARE
+    ddl_tag TEXT;
+  BEGIN
+    EXECUTE 'DELETE FROM cartodb.cdb_ddl_execution WHERE txid = txid_current() RETURNING tag;' INTO ddl_tag;
+    PERFORM cartodb.CDB_LinkGhostTables(ddl_tag);
+    RETURN NULL;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Event trigger to save the current transaction in cartodb.cdb_ddl_execution
+CREATE OR REPLACE FUNCTION cartodb.CDB_SaveDDLTransaction()
+RETURNS event_trigger
+AS $$
+  BEGIN
+    INSERT INTO cartodb.cdb_ddl_execution VALUES (txid_current(), tg_tag) ON CONFLICT (txid) DO NOTHING;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE SECURITY DEFINER;
+
+-- Creates the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION cartodb.CDB_EnableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+
+    -- Table to store the transaction id from DDL events to avoid multiple executions
+    CREATE TABLE IF NOT EXISTS cartodb.cdb_ddl_execution(txid integer PRIMARY KEY, tag text);
+
+    CREATE CONSTRAINT TRIGGER check_ddl_update
+    AFTER INSERT ON cartodb.cdb_ddl_execution
+    INITIALLY DEFERRED
+    FOR EACH ROW
+    EXECUTE PROCEDURE cartodb._CDB_LinkGhostTablesTrigger();
+
+    CREATE EVENT TRIGGER link_ghost_tables
+    ON ddl_command_end
+    WHEN TAG IN ('CREATE TABLE', 'SELECT INTO', 'DROP TABLE', 'ALTER TABLE', 'CREATE TRIGGER', 'DROP TRIGGER', 'CREATE VIEW', 'DROP VIEW', 'ALTER VIEW')
+    EXECUTE PROCEDURE cartodb.CDB_SaveDDLTransaction();
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;
+
+-- Drops the trigger on DDL events to link ghost tables
+CREATE OR REPLACE FUNCTION cartodb.CDB_DisableGhostTablesTrigger()
+RETURNS void
+AS $$
+  BEGIN
+    DROP EVENT TRIGGER IF EXISTS link_ghost_tables;
+    DROP TRIGGER IF EXISTS check_ddl_update ON cartodb.cdb_ddl_execution;
+    DROP TABLE IF EXISTS cartodb.cdb_ddl_execution;
+  END;
+$$ LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE;

scripts-available/CDB_GreatCircle.sql

@@ -16,7 +16,7 @@ BEGIN
 
   IF ST_XMax(line) - ST_XMin(line) > 180 THEN
     line = ST_Difference(
-      ST_Shift_Longitude(line),
+      ST_ShiftLongitude(line),
 			ST_Buffer(ST_GeomFromText('LINESTRING(180 90, 180 -90)', 4326), 0.00001)
 		);
   END IF;

scripts-available/CDB_TransformToWebmercator.sql

@@ -31,13 +31,17 @@ BEGIN
 
   -- Then we transform to WGS84 latlon, which is
   -- where we have known coordinates for the clipping
-  -- 
+  --
   latlon_input := ST_Transform(geom, 4326);
 
   -- Don't bother clipping if the geometry boundary doesn't
   -- go outside the valid extent.
   IF latlon_input @ valid_extent THEN
-    RETURN ST_Transform(latlon_input, 3857);
+    BEGIN
+      RETURN ST_Transform(latlon_input, 3857);
+    EXCEPTION WHEN OTHERS THEN
+      RETURN NULL;
+    END;
   END IF;
 
   -- Since we're going to use ST_Intersection on input
@@ -65,7 +69,7 @@ BEGIN
   -- We transform to web mercator
   to_webmercator := ST_Transform(clipped_input, 3857);
 
-  -- Finally we convert EMPTY to NULL 
+  -- Finally we convert EMPTY to NULL
   -- See https://github.com/Vizzuality/cartodb/issues/706
   -- And retain "multi" status
   ret := CASE WHEN ST_IsEmpty(to_webmercator) THEN NULL::geometry

scripts-available/CDB_Username.sql

@@ -0,0 +1,6 @@
+-- Returns the cartodb username of the current PostgreSQL session
+CREATE OR REPLACE FUNCTION cartodb.CDB_Username()
+RETURNS text
+AS $$
+  SELECT cartodb.CDB_Conf_GetConf(CONCAT('api_keys_', session_user))->>'username';
+$$ LANGUAGE SQL STABLE PARALLEL SAFE SECURITY DEFINER;

scripts-enabled/290-CDB_GhostTables.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_GhostTables.sql

scripts-enabled/900-CDB_DDLTriggers.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_DDLTriggers.sql

scripts-enabled/920-CDB_Username.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Username.sql

test/CDB_GhostTables.sql

@@ -0,0 +1,52 @@
+-- Create user and enable Ghost tables trigger
+\set QUIET on
+SET client_min_messages TO error;
+SELECT CDB_EnableGhostTablesTrigger();
+CREATE ROLE "fulano" LOGIN;
+GRANT ALL ON SCHEMA cartodb TO "fulano";
+GRANT SELECT ON cartodb.cdb_ddl_execution TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_Username() TO "fulano";
+GRANT EXECUTE ON FUNCTION CDB_LinkGhostTables(text) TO "fulano";
+SELECT cartodb.CDB_Conf_SetConf('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+DELETE FROM cdb_conf WHERE key = 'invalidation_service';
+SET SESSION AUTHORIZATION "fulano";
+SET client_min_messages TO notice;
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called (configuration not found)
+
+-- Add TIS configuration
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT cartodb.CDB_Conf_SetConf('invalidation_service', '{"host": "fake-tis-host", "port": 3142}');
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT CDB_LinkGhostTables(); -- _CDB_LinkGhostTables called
+
+BEGIN;
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- exists
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 0
+CREATE TABLE tmp(id INT);
+SELECT COUNT(*) FROM cartodb.cdb_ddl_execution; -- 1
+END; -- _CDB_LinkGhostTables called
+
+-- Disable Ghost tables trigger
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+SELECT CDB_DisableGhostTablesTrigger();
+SET SESSION AUTHORIZATION "fulano";
+\set QUIET off
+
+SELECT to_regclass('cartodb.cdb_ddl_execution'); -- not exists
+DROP TABLE tmp; -- _CDB_LinkGhostTables not called
+
+-- Cleanup
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE EXECUTE ON FUNCTION CDB_LinkGhostTables(text) FROM "fulano";
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM "fulano";
+REVOKE ALL ON SCHEMA cartodb FROM "fulano";
+DROP ROLE "fulano";
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano' OR key = 'invalidation_service';
+\set QUIET off

test/CDB_GhostTables_expect

@@ -0,0 +1,20 @@
+
+
+WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+
+BEGIN
+cdb_ddl_execution
+0
+CREATE TABLE
+1
+WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting fake-tis-host:3142. Name or service not known.
+NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+COMMIT
+
+
+DROP TABLE

test/CDB_Username.sql

@@ -0,0 +1,23 @@
+SELECT session_user; -- postgres
+SELECT CDB_Username(); -- (NULL)
+
+-- Add the role fulano with api_key and connect with it
+\set QUIET on
+CREATE ROLE fulano LOGIN;
+GRANT USAGE ON SCHEMA cartodb TO fulano;
+GRANT EXECUTE ON FUNCTION CDB_Username() TO fulano;
+INSERT INTO cdb_conf (key, value) VALUES ('api_keys_fulano', '{"username": "fulanito", "permissions":[]}');
+SET SESSION AUTHORIZATION fulano;
+\set QUIET off
+
+SELECT session_user; -- fulano
+SELECT CDB_Username(); -- fulanito
+
+-- Remove fulano
+\set QUIET on
+SET SESSION AUTHORIZATION postgres;
+REVOKE USAGE ON SCHEMA cartodb FROM fulano;
+REVOKE EXECUTE ON FUNCTION CDB_Username() FROM fulano;
+DROP ROLE fulano;
+DELETE FROM cdb_conf WHERE key = 'api_keys_fulano';
+\set QUIET off

test/CDB_Username_expect

@@ -0,0 +1,4 @@
+postgres
+
+fulano
+fulanito