Merge pull request #595 from CartoDB/development

Fix origin/destination in isolines

.github/workflows/main.yml

@@ -32,6 +32,10 @@ jobs:
       - name: Start docker-compose services
         run: docker-compose -f docker-compose.yaml up -d
 
+      - name: Install required python3 libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services"
+        if: env.PG_VERSION == 12
+
       - name: Install required python libs
         run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services"
 
@@ -44,9 +48,9 @@ jobs:
         timeout-minutes: 5
 
       - name: Run server tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)"
         timeout-minutes: 5
 
       - name: Run client tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)"
         timeout-minutes: 5

NEWS.md

@@ -1,3 +1,16 @@
+Mar 17th, 2020
+==============
+* Version `0.29.0` of the client extension
+    * Adapted client to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Version `0.38.0` of the server extension
+    * Adapted server to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Update dependencies to fix vulnerabilities
+
+Mar 6th, 2020
+==============
+* Version `0.23.1` of the Python library
+    * Added query per second throttle to TomTom bulk API calls
+
 Feb 12th, 2020
 ==============
 * Version `0.23.0` of the Python library

client/Makefile

@@ -14,6 +14,14 @@ AWK = awk
 PG_CONFIG         = pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
+
 # OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # DATA =  $(NEW_EXTENSION_ARTIFACT) \
 #   $(OLD_VERSIONS) \
@@ -25,9 +33,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 
-
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -54,7 +62,8 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR)/*.sql >> $@
+	cat $(SOURCES_DATA_DIR)/*.sql | \
+	$(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
 ifeq ($(PG_PARALLEL), 0)
 # Remove PARALLEL in aggregates and functions
 	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
@@ -87,6 +96,8 @@ devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 	rm -f $(GENERATED_SQL_FILES)
 
+clean: restore_copies
+
 # If needed remove PARALLEL tags from the release files
 release_remove_parallel_deploy:
 ifeq ($(PG_PARALLEL), 0)
@@ -97,9 +108,58 @@ ifeq ($(PG_PARALLEL), 0)
 	done
 endif
 
+restore_copies:
+	# tests
+	for f in $(basename $(wildcard test/sql/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	for f in $(basename $(wildcard test/expected/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f test/sql/*.copy;
+	rm -f test/expected/*.copy;
+	# data
+	for f in $(basename $(wildcard sql/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f sql/*.copy;
+	# old_versions
+	for f in $(basename $(wildcard old_versions/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f old_versions/*.copy;
+	# current scripts
+	for f in $(basename $(wildcard *.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f *.copy;
+
+# Replacing variables defined within test files
+replace_variables: restore_copies
+	# tests
+	for f in $(sort $(wildcard test/sql/*test.sql)); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	for f in $(sort $(wildcard test/expected/*test.out)); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# data
+	for f in $(wildcard sql/*.sql); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# old_versions
+	for f in $(wildcard old_versions/*.sql); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# current scripts
+	for f in $(wildcard *.sql); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $(EXTENSION).control;
+
 # Install the current release into the PostgreSQL extensions directory
 deploy: release_remove_parallel_deploy
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-install: deploy
+install: replace_variables deploy

client/carto-package.json

@@ -4,7 +4,7 @@
     "requires": {
       "postgresql": "^11.0.0",
       "postgis": "^2.5.0.0",
-      "carto_postgresql_ext": "^0.32.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
     }

client/cdb_dataservices_client--0.28.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/cdb_dataservices_client--0.29.0--0.28.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.28.0'
+default_version = '0.29.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/sql/05_utils.sql

@@ -17,4 +17,17 @@ $func$ LANGUAGE plpgsql;
 -- Taken from https://stackoverflow.com/a/48013356/351721
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
     SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
-$f$ LANGUAGE sql IMMUTABLE;
+$f$ LANGUAGE sql IMMUTABLE;
+
+
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/sql/20_table_augmentation.sql

@@ -138,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeas
         )
 
     return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
     username text,
@@ -231,7 +231,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMea
             fdw_server=plpy.quote_literal(server_name)))
 
     return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
     username text,

client/test/expected/00_installation_test.out

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;
 -- Install the extension

client/test/sql/00_installation_test.sql

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;
 

server/extension/Makefile

@@ -5,7 +5,6 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/defau
 # The new version to be generated from templates
 SED = sed
 ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
 NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
 AWK = awk
 
@@ -14,7 +13,8 @@ PG_CONFIG         = pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -33,6 +33,19 @@ PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
+
+REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
+ifneq ($(NEW_VERSION),)
+REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
+endif
+
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
@@ -44,22 +57,27 @@ ifeq ($(PG_PARALLEL), 0)
 	mv $(TMPFILE) $@
 endif
 
+# $(EXTENSION).control: $(EXTENSION).control.in Makefile
+# 	$(SED) $(REPLACEMENTS)  $< > $@
+
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
 release: $(EXTENSION).control $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
-	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	for f in $(wildcard *.sql); do \
+		git mv $${f} old_versions/$${f}; \
+	done
+	$(SED) -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' $(EXTENSION).control
 	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
 	@echo
-	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
-	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
+	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql.in and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
+	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql.in and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
 	@echo
 
 # Only meant for development time, do not use once a version is released
@@ -67,6 +85,8 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 
+clean: restore_copies
+
 # If needed remove PARALLEL tags from the release files
 release_remove_parallel_deploy:
 ifeq ($(PG_PARALLEL), 0)
@@ -82,7 +102,60 @@ deploy: release_remove_parallel_deploy
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-install: deploy
+restore_copies:
+	# tests
+	for f in $(basename $(wildcard test/sql/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	for f in $(basename $(wildcard test/expected/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f test/sql/*.copy;
+	rm -f test/expected/*.copy;
+	# data
+	for f in $(basename $(wildcard sql/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f sql/*.copy;
+	# old_versions
+	for f in $(basename $(wildcard old_versions/*.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f old_versions/*.copy;
+	# current scripts
+	for f in $(basename $(wildcard *.copy)); do \
+		cat $${f}.copy > $${f}; \
+	done
+	rm -f *.copy;
+
+
+# %.sql:
+# 	$(SED) $(REPLACEMENTS)  $< > $@
+
+# Replacing variables defined within test files
+replace_variables: restore_copies
+	# tests
+	for f in $(sort $(wildcard test/sql/*test.sql)); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	for f in $(sort $(wildcard test/expected/*test.out)); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# data
+	for f in $(SOURCES_DATA); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# old_versions
+	for f in $(wildcard old_versions/*.sql); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	# current scripts
+	for f in $(wildcard *.sql); do \
+		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
+	done
+	sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $(EXTENSION).control;
+
+install: replace_variables deploy
 
 reinstall: install
 	psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"

server/extension/carto-package.json

@@ -4,11 +4,11 @@
     "requires": {
       "postgresql": "^10.0.0",
       "postgis": "^2.4.0.0",
-      "carto_postgresql_ext": "^0.23.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
-      "dataservices-api-server-python-lib": "^0.19.1",
+      "dataservices-api-server-python-lib": "^0.23.1",
       "observatory-server-extension": "^1.9.0"
     }
   }
-}
+}

server/extension/cdb_dataservices_server--0.37.0--0.38.0.sql

@@ -0,0 +1,76 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.38.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_server.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.38.0--0.37.0.sql

@@ -0,0 +1,71 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_server.geomval RESTRICT;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.37.0'
-requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
+default_version = '0.38.0'
+requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/05_postgis3_adapter.sql

@@ -0,0 +1,11 @@
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_server.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

server/extension/sql/100_routing_helper.sql

@@ -69,7 +69,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate Mapbox routing')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
   username TEXT,
@@ -137,7 +137,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate TomTom routing')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
   username TEXT,
@@ -198,4 +198,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate mapzen routing')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/105_route_between_points.sql

@@ -37,7 +37,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
@@ -76,4 +76,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/10_redis_helper.sql

@@ -15,4 +15,4 @@ RETURNS boolean AS $$
       'redis_metrics_connection': redis_metrics_connection,
     }
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/110_data_observatory_augmentation.sql

@@ -14,7 +14,7 @@ RETURNS text AS $$
   user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   return user_obs_config.connection_str
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
   username TEXT,
@@ -68,7 +68,7 @@ RETURNS json AS $$
         raise Exception('Error trying to obs_get_demographic_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
   username TEXT,
@@ -125,7 +125,7 @@ RETURNS SETOF JSON AS $$
         raise Exception('Error trying to obs_get_demographic_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
   username TEXT,
@@ -177,7 +177,7 @@ RETURNS json AS $$
         raise Exception('Error trying to obs_get_segment_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
   username TEXT,
@@ -232,7 +232,7 @@ RETURNS SETOF JSON AS $$
         raise Exception('Error trying to OBS_GetSegmentSnapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
   username TEXT,
@@ -289,7 +289,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetMeasure')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
   username TEXT,
@@ -344,7 +344,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetCategory')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
   username TEXT,
@@ -401,7 +401,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetUSCensusMeasure')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
   username TEXT,
@@ -456,7 +456,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetUSCensusCategory')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
   username TEXT,
@@ -511,7 +511,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetPopulation')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
   username TEXT,
@@ -566,7 +566,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetMeasureById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
   username TEXT,
@@ -627,7 +627,7 @@ RETURNS TABLE (
         raise Exception('Error trying to OBS_GetData')
     finally:
         quota_service.increment_total_service_use(len(geomvals))
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
   username TEXT,
@@ -688,7 +688,7 @@ RETURNS TABLE (
         raise Exception('Error trying to OBS_GetData')
     finally:
         quota_service.increment_total_service_use(len(geomrefs))
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
   username TEXT,
@@ -736,7 +736,7 @@ RETURNS JSON AS $$
         import sys
         logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
         raise Exception('Error trying to OBS_GetMeta')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation(
   username TEXT,
@@ -782,4 +782,4 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$
         import sys
         logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname})
         raise Exception('Error trying to OBS_MetadataValidation')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/115_data_observatory_exploration.sql

@@ -55,7 +55,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
         raise Exception('Error trying to OBS_Search')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
   username TEXT,
@@ -113,4 +113,4 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
         raise Exception('Error trying to OBS_GetMeasureById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/120_data_observatory_geometries.sql

@@ -49,7 +49,7 @@ RETURNS geometry(Geometry, 4326) AS $$
         raise Exception('Error trying to OBS_GetBoundary')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
   username TEXT,
@@ -102,7 +102,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetBoundaryId')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
   username TEXT,
@@ -155,7 +155,7 @@ RETURNS geometry(Geometry, 4326) AS $$
         raise Exception('Error trying to OBS_GetBoundaryById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
   username TEXT,
@@ -215,7 +215,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetBoundariesByGeometry')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
   username TEXT,
@@ -277,7 +277,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
   username TEXT,
@@ -337,7 +337,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetPointsByGeometry')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
   username TEXT,
@@ -399,4 +399,4 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/125_data_observatory_table_augment.sql

@@ -8,7 +8,7 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
     return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
         .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
         )[0]
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
 RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$

server/extension/sql/15_config_helper.sql

@@ -8,7 +8,7 @@ RETURNS boolean AS $$
     logger_config = LoggerConfig(plpy)
     GD[cache_key] = logger_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 -- This is done in order to avoid an undesired depedency on cartodb extension
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
@@ -46,7 +46,7 @@ RETURNS boolean AS $$
     geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
     GD[cache_key] = geocoder_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -60,7 +60,7 @@ RETURNS boolean AS $$
     geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = geocoder_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -74,7 +74,7 @@ RETURNS boolean AS $$
     isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = isolines_routing_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -88,7 +88,7 @@ RETURNS boolean AS $$
     routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = routing_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -102,4 +102,4 @@ RETURNS boolean AS $$
     obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = obs_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/200_quotas.sql

@@ -106,7 +106,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
   ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
 
   return ret
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
@@ -136,7 +136,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
       ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
 
   return ret
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
   username TEXT,

server/extension/sql/20_geocode_street.sql

@@ -36,7 +36,7 @@ RETURNS Geometry AS $$
     else:
       raise Exception('Requested geocoder is not available')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
@@ -52,7 +52,7 @@ RETURNS Geometry AS $$
   else:
     raise Exception('Here geocoder is not available for your account.')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -67,7 +67,7 @@ RETURNS Geometry AS $$
   else:
     raise Exception('Google geocoder is not available for your account.')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -80,7 +80,7 @@ RETURNS Geometry AS $$
   mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -93,7 +93,7 @@ RETURNS Geometry AS $$
   mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -106,7 +106,7 @@ RETURNS Geometry AS $$
   tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -119,7 +119,7 @@ RETURNS Geometry AS $$
   geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -152,7 +152,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using here maps')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -184,7 +184,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using google maps')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -225,7 +225,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using mapzen')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -271,7 +271,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using Mapbox')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -317,7 +317,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using TomTom')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -363,4 +363,4 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using Geocodio')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/210_rates.sql

@@ -16,7 +16,7 @@ RETURNS JSON AS $$
       return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
   else:
       return None
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
   username TEXT,
@@ -40,7 +40,7 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_user_rate_limits(config)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
   username TEXT,
@@ -64,7 +64,7 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_org_rate_limits(config)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
   username TEXT,
@@ -88,4 +88,4 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_server_rate_limits(config)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;

server/extension/sql/21_bulk_geocode_street.sql

@@ -40,7 +40,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
     plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
     return plpy.execute(plan, [username, orgname, searches])
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -51,7 +51,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
   geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
   geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
   geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -96,7 +96,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
   geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -113,4 +113,4 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
   geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/30_admin0.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode admin0 polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 --------------------------------------------------------------------------------

server/extension/sql/40_admin1.sql

@@ -36,7 +36,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode admin1 polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
@@ -74,7 +74,7 @@ RETURNS Geometry AS $$
         raise Exception('Error trying to geocode admin1 polygon')
       finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/50_namedplaces.sql

@@ -1,7 +1,7 @@
 ---- cdb_geocode_namedplace_point(city_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -16,12 +16,12 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_namedplace_point(city_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -36,12 +36,12 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -56,7 +56,7 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -101,7 +101,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode namedplace point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/60_postalcodes.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision)
 RETURNS Geometry AS $$
@@ -77,7 +77,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text)
 RETURNS Geometry AS $$
@@ -119,7 +119,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision)
 RETURNS Geometry AS $$
@@ -162,7 +162,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text)
 RETURNS Geometry AS $$

server/extension/sql/70_ips.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/80_isolines_helper.sql

@@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isolines')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
    username TEXT,
@@ -121,7 +121,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isolines')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
    username TEXT,
@@ -185,7 +185,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get Mapbox isolines')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
    username TEXT,
@@ -249,7 +249,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get TomTom isolines')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
    username TEXT,
@@ -310,7 +310,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isochrones')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
    username TEXT,
@@ -368,7 +368,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get Mapbox isochrones')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
    username TEXT,
@@ -426,4 +426,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get TomTom isochrones')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/85_isodistance.sql

@@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- heremaps isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapzen isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -60,7 +60,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapbox isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -74,7 +74,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- tomtom isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -88,4 +88,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/90_isochrone.sql

@@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- heremaps isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapzen isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -59,7 +59,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapbox isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -72,7 +72,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- tomtom isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -85,4 +85,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/test/expected/00_install_test.out

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;

server/extension/test/expected/100_data_observatory_test.out

@@ -1,3 +1,4 @@
+\set ECHO none
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/00_install_test.sql

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;

server/extension/test/sql/100_data_observatory_test.sql

@@ -1,3 +1,10 @@
+\set ECHO none
+-- add the schema cdb_dataservices_server to the SEARCH_PATH
+DO $$ BEGIN
+    PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
+END $$;
+\set ECHO all
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/366_empty_table_test.sql

@@ -2,6 +2,11 @@
 \set VERBOSITY verbose
 SET client_min_messages TO error;
 
+-- add the schema cdb_dataservices_server to the SEARCH_PATH
+DO $$ BEGIN
+    PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
+END $$;
+
 -- Set configuration for a user 'foo'
 DO $$
   import json
@@ -14,7 +19,7 @@ DO $$
   service_config.user.set('soft_obs_general_limit', True)
   service_config.user.set('period_end_date', '20170516')
 
-$$ LANGUAGE plpythonu;
+$$ LANGUAGE @@plpythonu@@;
 
 
 -- Mock Observatory backend function
@@ -37,7 +42,7 @@ GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api;
 SELECT * FROM cdb_dataservices_server.OBS_GetData(
     'foo',
     NULL,
-    '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval,
+    '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval,
     '[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": '
          '"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments'
          'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries'

server/lib/python/cartodb_services/cartodb_services/__init__.py

@@ -34,4 +34,4 @@ def _reset():
     plpy = None
     GD = None
 
-from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
+from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -1,8 +1,8 @@
-from google import GoogleMapsBulkGeocoder
-from here import HereMapsBulkGeocoder
-from tomtom import TomTomBulkGeocoder
-from mapbox import MapboxBulkGeocoder
-from geocodio import GeocodioBulkGeocoder
+from cartodb_services.google import GoogleMapsBulkGeocoder
+from cartodb_services.here import HereMapsBulkGeocoder
+from cartodb_services.tomtom import TomTomBulkGeocoder
+from cartodb_services.mapbox import MapboxBulkGeocoder
+from cartodb_services.geocodio import GeocodioBulkGeocoder
 
 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
     'google': GoogleMapsBulkGeocoder,

server/lib/python/cartodb_services/cartodb_services/config/__init__.py

@@ -1,3 +1,3 @@
-from service_configuration import ServiceConfiguration
-from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
-from legacy_rate_limits import RateLimitsConfigLegacyBuilder
+from .service_configuration import ServiceConfiguration
+from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+from .legacy_rate_limits import RateLimitsConfigLegacyBuilder

server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py

@@ -1,5 +1,5 @@
 import json
-from rate_limits import RateLimitsConfig
+from cartodb_services.config.rate_limits import RateLimitsConfig
 
 class RateLimitsConfigLegacyBuilder(object):
     """

server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py

@@ -1,6 +1,6 @@
 import json
 
-from service_configuration import ServiceConfiguration
+from cartodb_services.config.service_configuration import ServiceConfiguration
 
 class RateLimitsConfig(object):
     """

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -1,7 +1,7 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
-from tools import QuotaExceededException, Logger
+from cartodb_services.tools import QuotaExceededException, Logger
 from collections import namedtuple
 import json
 
@@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
                 except Exception as e:
                     import sys
                     logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
-                    metadata['processing_error'] = 'Error: {}'.format(e.message)
+                    metadata['processing_error'] = 'Error: {}'.format(e)
                     results.append([result[0], None, json.dumps(metadata)])
                     failed_count += 1
 

server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GeocodioGeocoder
-from bulk_geocoder import GeocodioBulkGeocoder
+from cartodb_services.geocodio.geocoder import GeocodioGeocoder
+from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GoogleMapsGeocoder
-from bulk_geocoder import GoogleMapsBulkGeocoder
+from cartodb_services.google.geocoder import GoogleMapsGeocoder
+from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py

@@ -1,5 +1,5 @@
 from multiprocessing import Pool
-from exceptions import MalformedResult
+from cartodb_services.google.exceptions import MalformedResult
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.geocoder import compose_address, geocoder_error_response
 from cartodb_services.google import GoogleMapsGeocoder

server/lib/python/cartodb_services/cartodb_services/google/client_factory.py

@@ -3,7 +3,7 @@
 
 import googlemaps
 import base64
-from exceptions import InvalidGoogleCredentials
+from cartodb_services.google.exceptions import InvalidGoogleCredentials
 
 
 class GoogleMapsClientFactory():

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,9 +1,11 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
+try:
+    from urlparse import parse_qs
+except:
+    from urllib.parse import parse_qs
 
-from urlparse import parse_qs
-
-from exceptions import MalformedResult
+from cartodb_services.google.exceptions import MalformedResult
 from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
 from cartodb_services.google.exceptions import InvalidGoogleCredentials
 from client_factory import GoogleMapsClientFactory

server/lib/python/cartodb_services/cartodb_services/here/__init__.py

@@ -1,3 +1,3 @@
-from geocoder import HereMapsGeocoder
-from bulk_geocoder import HereMapsBulkGeocoder
-from routing import HereMapsRoutingIsoline
+from cartodb_services.here.geocoder import HereMapsGeocoder
+from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
+from cartodb_services.here.routing import HereMapsRoutingIsoline

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -1,7 +1,7 @@
 import requests
 import json
 
-from exceptions import WrongParams
+from cartodb_services.here.exceptions import WrongParams
 from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 
@@ -115,7 +115,7 @@ class HereMapsRoutingIsoline(Traceable):
 
     def __parse_source_param(self, source, options):
         key = 'start'
-        if 'is_destination' in options and options['is_destination']:
+        if 'is_destination' in options and options['is_destination'].lower() == 'true':
             key = 'destination'
 
         return {key: source}

server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py

@@ -1,4 +1,4 @@
-from routing import MapboxRouting, MapboxRoutingResponse
-from geocoder import MapboxGeocoder
-from bulk_geocoder import MapboxBulkGeocoder
-from isolines import MapboxIsolines, MapboxIsochronesResponse
+from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
+from cartodb_services.mapbox.geocoder import MapboxGeocoder
+from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
+from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -1,5 +1,5 @@
-from routing import MapzenRouting, MapzenRoutingResponse
-from isolines import MapzenIsolines
-from geocoder import MapzenGeocoder
-from matrix_client import MatrixClient
-from isochrones import MapzenIsochrones
+from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
+from cartodb_services.mapzen.isolines import MapzenIsolines
+from cartodb_services.mapzen.geocoder import MapzenGeocoder
+from cartodb_services.mapzen.matrix_client import MatrixClient
+from cartodb_services.mapzen.isochrones import MapzenIsochrones

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -135,7 +135,7 @@ class MapzenIsolines:
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c <> isorange:
+            if c != isorange:
                 location_estimates_filtered.append(location_estimates[i])
 
         return location_estimates_filtered

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
-from quota import QuotaService
-from user import UserMetricsService
-from log import metrics, MetricsDataGatherer, Traceable
+from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
+from cartodb_services.metrics.quota import QuotaService
+from cartodb_services.metrics.user import UserMetricsService
+from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -45,7 +45,7 @@ class ServiceConfig(object):
 
     def _get_effective_monthly_quota(self, quota_key, default=0):
         quota_from_redis = self._redis_config.get(quota_key, None)
-        if quota_from_redis and quota_from_redis <> '':
+        if quota_from_redis and quota_from_redis != '':
             return int(quota_from_redis)
         else:
             return default

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -6,7 +6,10 @@ import uuid
 import plpy
 from datetime import datetime
 from contextlib import contextmanager
-from urlparse import urlparse
+try:
+    from urlparse import urlparse
+except:
+    from urllib.parse import urlparse
 
 
 @contextmanager

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -1,5 +1,5 @@
-from user import UserMetricsService
-from log import MetricsDataGatherer
+from cartodb_services.metrics.user import UserMetricsService
+from cartodb_services.metrics.log import MetricsDataGatherer
 from datetime import date
 import re
 

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -1,5 +1,5 @@
 from ..core.interfaces import ConfigBackendInterface
-from null_config import NullConfigStorage
+from cartodb_services.refactor.storage.null_config import NullConfigStorage
 
 
 class RedisConfigStorage(ConfigBackendInterface):

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py

@@ -18,5 +18,5 @@ class RedisConnectionBuilder():
         else:
             conn = StrictRedis(host=self._config.host, port=self._config.port,
                                db=self._config.db, retry_on_timeout=True,
-                               socket_timeout=self._config.timeout)
+                               socket_timeout=self._config.timeout, decode_responses=True)
             return conn

server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py

@@ -1,4 +1,4 @@
-from geocoder import TomTomGeocoder
-from bulk_geocoder import TomTomBulkGeocoder
-from routing import TomTomRouting, TomTomRoutingResponse
-from isolines import TomTomIsolines, TomTomIsochronesResponse
+from cartodb_services.tomtom.geocoder import TomTomGeocoder
+from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
+from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
+from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py

@@ -3,6 +3,7 @@ from requests.adapters import HTTPAdapter
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.geocoder import geocoder_error_response
 from cartodb_services.tomtom import TomTomGeocoder
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools.exceptions import ServiceException
 
 
@@ -59,6 +60,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
             self._logger.error(msg, e)
             return [geocoder_error_response(msg)] * len(searches)
 
+    @qps_retry(qps=5, provider='tomtom')
     def _send_batch(self, searches):
         body = {'batchItems': [{'query': self._query(s)} for s in searches]}
         request_params = {
@@ -76,6 +78,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
             self._logger.error(msg)
             raise ServiceException(msg, response)
 
+    @qps_retry(qps=5, provider='tomtom')
     def _download_results(self, location):
         stalled_retries = 0
         while True:

server/lib/python/cartodb_services/cartodb_services/tools/__init__.py

@@ -1,9 +1,9 @@
-from redis_tools import RedisConnection, RedisDBConfig
-from coordinates import Coordinate
-from polyline import PolyLine
-from log import Logger, LoggerConfig
-from rate_limiter import RateLimiter
-from service_manager import ServiceManager
-from legacy_service_manager import LegacyServiceManager
-from exceptions import QuotaExceededException, RateLimitExceeded
-from country import country_to_iso3
+from .redis_tools import RedisConnection, RedisDBConfig
+from .coordinates import Coordinate
+from .polyline import PolyLine
+from .log import Logger, LoggerConfig
+from .rate_limiter import RateLimiter
+from .service_manager import ServiceManager
+from .legacy_service_manager import LegacyServiceManager
+from .exceptions import QuotaExceededException, RateLimitExceeded
+from .country import country_to_iso3

server/lib/python/cartodb_services/cartodb_services/tools/qps.py

@@ -1,7 +1,7 @@
 import time
 import random
 from datetime import datetime
-from exceptions import TimeoutException
+from cartodb_services.tools.exceptions import TimeoutException
 import re
 
 DEFAULT_RETRY_TIMEOUT = 60

server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py

@@ -23,7 +23,7 @@ class RedisConnection:
         else:
             conn = StrictRedis(host=self._config.host, port=self._config.port,
                                db=self._config.db, retry_on_timeout=True,
-                               socket_timeout=self._config.timeout)
+                               socket_timeout=self._config.timeout, decode_responses=True)
             return conn
 
 

server/lib/python/cartodb_services/requirements.txt

@@ -4,7 +4,7 @@ python-dateutil==2.2
 googlemaps==2.5.1
 rollbar==0.13.2
 # Dependency for googlemaps package
-requests==2.9.1
+requests==2.20.0
 rratelimit==0.0.4
 mapbox==0.14.0
 pygeocodio==0.11.1

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.23.0',
+    version='0.23.2',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_heremapsrouting.py

@@ -204,6 +204,16 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
                          'fastest;car;traffic:false;motorway:-1')
 
     def test_source_parameters_works_properly(self, req_mock):
+        req_mock.register_uri('GET', requests_mock.ANY,
+                              text=self.GOOD_RESPONSE)
+        response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',
+                                                    ['1000', '2000'],
+                                                    ['is_destination=false'])
+        parsed_url = urlparse(req_mock.request_history[0].url)
+        url_params = parse_qs(parsed_url.query)
+        self.assertEqual(url_params['start'][0], 'geo!33.0,1.0')
+
+    def test_destination_parameters_works_properly(self, req_mock):
         req_mock.register_uri('GET', requests_mock.ANY,
                               text=self.GOOD_RESPONSE)
         response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',

test/helpers/import_helper.py

@@ -20,7 +20,7 @@ class ImportHelper:
         response.raise_for_status()
         response_json = json.loads(response.text)
         if not response_json['success']:
-            print "Error importing the test dataset: {0}".format(response.text)
+            print("Error importing the test dataset: {0}".format(response.text))
             sys.exit(1)
         while(True):
             table_name = ImportHelper.get_imported_table_name(
@@ -42,8 +42,8 @@ class ImportHelper:
             schema, username, host, import_id, api_key)
         import_data_response = requests.get(import_url)
         if import_data_response.status_code != 200:
-            print "Error getting the table name from " \
-                "the import data: {0}".format(import_data_response.text)
+            print("Error getting the table name from " \
+                "the import data: {0}".format(import_data_response.text))
             sys.exit(1)
         import_data_json = json.loads(import_data_response.text)
 
@@ -57,5 +57,5 @@ class ImportHelper:
         )
         response = requests.get(url)
         if response.status_code != 200:
-            print "Error cleaning the test dataset: {0}".format(response.text)
+            print("Error cleaning the test dataset: {0}".format(response.text))
             sys.exit(1)

test/helpers/integration_test_helper.py

@@ -10,7 +10,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
     return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
 
 
-def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
+def assert_close_enough(xy_a, xy_b, rel_tol=0.001, abs_tol=0.0005):
     """
     Asserts that the given points are "close enough", in a square.
     :param xy_a: Array of 2 elements, X and Y.
@@ -23,7 +23,7 @@ def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
         assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol),
                     "Coord {} error: {} and {} are not closer than {}, {}".format(
                         i, xy_a[i], xy_b[i], rel_tol, abs_tol
-                    ))
+        ))
 
 
 class IntegrationTestHelper:
@@ -45,11 +45,15 @@ class IntegrationTestHelper:
         }
 
     @classmethod
-    def execute_query_raw(cls, sql_api_url, query):
+    def execute_query_raw(cls, sql_api_url, query, method='GET'):
         requests.packages.urllib3.disable_warnings()
-        query_url = "{0}?q={1}".format(sql_api_url, query)
-        print "Executing query: {0}".format(query_url)
-        query_response = requests.get(query_url)
+        if method.upper() == 'GET':
+            query_url = "{0}?q={1}".format(sql_api_url, query)
+            query_response = requests.get(query_url)
+            print("Executing query GET: {0}".format(query_url))
+        else:
+            query_response = requests.post(sql_api_url, data={"q": query})
+            print("Executing query POST: {0}".format(sql_api_url))
         if query_response.status_code != 200:
             raise Exception(json.loads(query_response.text)['error'])
         return json.loads(query_response.text)

test/integration/test_street_functions.py

@@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase):
 
     TOMTOM_METADATAS = {
         'Plaza España, Barcelona':
-            {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']},
+            {'relevance': 0.72, 'precision': 'precise', 'match_types': ['street']},
         'Santiago Rusiñol 123, Valladolid':
-            {'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']}
+            {'relevance': 0.74, 'precision': 'precise', 'match_types': ['street']}
     }
 
     MAPBOX_METADATAS = {
@@ -151,10 +151,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp):
         query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \
                 "as geometry FROM {0} LIMIT 1".format(table)
         try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+            self._run_authenticated(query)['rows'][0]
         except Exception as e:
-            assert_equal(e.message[0],
-                         "permission denied for relation {}".format(table))
+            assert_equal(e, "permission denied for relation {}".format(table))
 
     def test_component_aggregation(self):
         query = "select st_x(the_geom), st_y(the_geom) from (" \
@@ -335,7 +334,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
                 "{}" \
                 "]''::jsonb) as (cartodb_id integer, address text)', " \
                 "'address', null, null, null, {})".format(','.join(streets), batch_size)
-        response = self._run_authenticated(query)
+        response = self._run_authenticated(query, method='POST')
         assert_equal(n, len(response['rows']))
         for row in response['rows']:
             assert_not_equal(row['st_x'], None)
@@ -413,12 +412,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
         for r, e in zip(response['rows'], expected):
             self.assert_metadata(r['metadata'], e)
 
-    def _run_authenticated(self, query):
-        authenticated_query = "{}&api_key={}".format(query,
-                                                     self.env_variables[
-                                                         'api_key'])
-        return IntegrationTestHelper.execute_query_raw(self.sql_api_url,
-                                                       authenticated_query)
+    def _run_authenticated(self, query, method='GET'):
+        api_key = self.env_variables['api_key']
+        url = self.sql_api_url
+        auth_query = "{}&api_key={}".format(query, api_key)
+        if method.upper() != 'GET':
+            auth_query = query
+            url = "{}?api_key={}".format(self.sql_api_url, api_key)
+        return IntegrationTestHelper.execute_query_raw(url, auth_query, method)
 
     @staticmethod
     def _x_y_by_cartodb_id(response):

test/requirements.txt

@@ -1,3 +1,3 @@
 # Integration tests dependencies
-requests==2.9.1
+requests==2.20.0
 nose==1.3.7

test/run_tests.py

@@ -36,7 +36,7 @@ def main():
         set_environment_variables(username, api_key, table_name, host, schema)
         execute_tests()
     except Exception as e:
-        print e.message
+        print(e)
         sys.exit(1)
     finally:
         clean_environment_variables()
@@ -45,11 +45,11 @@ def main():
 
 
 def usage():
-    print """Usage: run_tests.py [options] username api_key
+    print("""Usage: run_tests.py [options] username api_key
         Options:
         -h: Show this help
         --host: take that host as base (by default is cartodb.com)
-        --schema: define the url schema [http/https] (by default https)"""
+        --schema: define the url schema [http/https] (by default https)""")
 
 
 def execute_tests():
@@ -59,7 +59,7 @@ def execute_tests():
         stderr=subprocess.PIPE
     )
     out, err = process.communicate()
-    print err
+    print(err.decode('utf-8'))
     regexp = re.compile(r'FAILED \(.*\)')
     if regexp.search(err) is not None:
         sys.exit(1)