Merge pull request #392 from Algunenano/test_bionic

Test bionic

.travis.yml

@@ -1,4 +1,3 @@
-dist: xenial
 language: c
 sudo: required
 
@@ -12,10 +11,15 @@ env:
 jobs:
   include:
     - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+      dist: xenial
     - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+      dist: xenial
     - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
+      dist: xenial
     - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
+      dist: bionic
     - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
+      dist: bionic
 
 script:
   - sudo service postgresql stop;

Makefile

@@ -150,7 +150,7 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	cat $(CDBSCRIPTS) | \
 	$(SED) 	-e 's/@extschema@/cartodb/g' \
 		-e 's/@postgisschema@/public/g' \
-		-e 's/plpythonu/$(PLPYTHONU)/g' >> $@
+		-e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@
 
@@ -164,10 +164,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 	cp $< $@
 
 $(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
 
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
 
 # Needed for consistent `echo` results with backslashes
 SHELL = bash

cartodb.control.in

@@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
 superuser = true
 relocatable = false
 schema = cartodb
-requires = 'plpythonu, postgis'
+requires = '@@plpythonu@@, postgis'

scripts-available/CDB_FederatedServerDiagnostics.sql

@@ -191,7 +191,7 @@ AS $$
         })
     return json.dumps(stats)
 $$
-LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 
 --

scripts-available/CDB_GhostTables.sql

@@ -32,7 +32,6 @@ AS $$
           client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
           GD['invalidation'] = client
         except Exception as err:
-          error = "client_error - %s" % str(err)
           # NOTE: no retries on connection error
           plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' +  str(err))
           break
@@ -41,13 +40,12 @@ AS $$
       client.execute_command('DBSCH', db_name, username, event_name)
       break
     except Exception as err:
-      error = "request_error - %s" % str(err)
       client = GD['invalidation'] = None # force reconnect
       if not tis_retry:
         plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' +  str(err))
         break
       tis_retry -= 1 # try reconnecting
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 
 -- Enqueues a job to run Ghost tables linking process for the current user
 CREATE OR REPLACE FUNCTION @extschema@.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
@@ -61,7 +59,7 @@ AS $$
     EXECUTE 'SELECT current_database();' INTO db_name;
 
     PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
-    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+    RAISE INFO '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
   END;
 $$  LANGUAGE plpgsql
     VOLATILE

scripts-available/CDB_Groups_API.sql

@@ -21,7 +21,7 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -41,7 +41,7 @@ $$
 
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -61,7 +61,7 @@ $$
     body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -81,7 +81,7 @@ $$
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -101,7 +101,7 @@ $$
     body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -129,7 +129,7 @@ $$
     body = '{ "access": "%s" }' % access
     query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -156,7 +156,7 @@ $$
     url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
     query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
     plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
     VOLATILE
     PARALLEL UNSAFE
     SECURITY DEFINER
@@ -191,7 +191,7 @@ $$
       params = json.loads(conf)
       auth = 'Basic %s' % plpy.execute("SELECT @extschema@._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
       return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
@@ -199,7 +199,7 @@ FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
 $$
     import base64
     return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 
 -- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
 CREATE OR REPLACE
@@ -239,5 +239,5 @@ $$
       raise last_err
 
     return None
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 revoke all on function @extschema@._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_QueryStatements.sql

@@ -11,4 +11,4 @@ RETURNS SETOF TEXT AS $$
     cleaned = match[0].strip()
     if ( cleaned ):
       yield cleaned
-$$ language 'plpythonu' IMMUTABLE STRICT PARALLEL SAFE;
+$$ language '@@plpythonu@@' IMMUTABLE STRICT PARALLEL SAFE;

test/CDB_GhostTables.sql

@@ -1,6 +1,56 @@
 -- Create user and enable Ghost tables trigger
 \set QUIET on
 SET client_min_messages TO error;
+
+-- Recreate the function without extra error messages as it changes depending on the python-redis version
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text)
+RETURNS void
+AS $$
+  if not username:
+    return
+
+  if 'json' not in GD:
+    import json
+    GD['json'] = json
+  else:
+    json = GD['json']
+
+  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
+  if not tis_config:
+    plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
+    return
+
+  tis_config_dict = json.loads(tis_config)
+  tis_host = tis_config_dict.get('host')
+  tis_port = tis_config_dict.get('port')
+  tis_timeout = tis_config_dict.get('timeout', 5)
+  tis_retry = tis_config_dict.get('retry', 5)
+
+  client = GD.get('invalidation', None)
+
+  while True:
+
+    if not client:
+        try:
+          import redis
+          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
+          GD['invalidation'] = client
+        except Exception as err:
+          # NOTE: no retries on connection error
+          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables')
+          break
+
+    try:
+      client.execute_command('DBSCH', db_name, username, event_name)
+      break
+    except Exception as err:
+      client = GD['invalidation'] = None # force reconnect
+      if not tis_retry:
+        plpy.warning('Error calling Invalidation Service to link Ghost Tables')
+        break
+      tis_retry -= 1 # try reconnecting
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+
 SELECT CDB_EnableGhostTablesTrigger();
 CREATE ROLE "fulano" LOGIN;
 GRANT ALL ON SCHEMA cartodb TO "fulano";

test/CDB_GhostTables_expect

@@ -1,19 +1,19 @@
 
 
 WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
 
 BEGIN
 cdb_ddl_execution
 0
 CREATE TABLE
 1
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
+WARNING:  Error calling Invalidation Service to link Ghost Tables
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
 COMMIT