Merge pull request #616 from CartoDB/development

Preparing release (server 0.39.3 python 0.23.5)

.github/workflows/main.yml

@@ -4,6 +4,7 @@ jobs:
   dataservices-api:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         pg_version: [10, 12]
     env:
@@ -11,6 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
+      - name: Set CLOUDSDK_PYTHON path
+        run: echo "CLOUDSDK_PYTHON=/usr/bin/python" >> $GITHUB_ENV
+
       - name: Setup gcloud authentication
         uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
         with:
@@ -48,9 +52,9 @@ jobs:
         timeout-minutes: 5
 
       - name: Run server tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test/regression.diffs && false)"
         timeout-minutes: 5
 
       - name: Run client tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /dataservices-api/client/test/regression.diffs && false)"
         timeout-minutes: 5

NEWS.md

@@ -1,3 +1,42 @@
+Jan 26th, 2021
+==============
+* Version `0.39.3` of the server extension
+* Version `0.23.5` of the Python library
+    * Country name conversion to ISO code for TomTom bulk geocoding.
+
+Nov 19th, 2020
+==============
+* Version `0.39.2` of the server extension
+* Version `0.23.4` of the Python library
+    * Better messages on remote errors.
+    * Fix error on unknown status codes.
+    * Here bulk geocoder: Return also errors and no-matches.
+
+Sep 23th, 2020
+==============
+* Version `0.39.1` of the server extension
+    * Allow isolines services for google-geocoding users.
+
+Sep 23th, 2020
+==============
+* Version `0.23.3` of the Python library
+    * Change HERE batch geocoding URL
+
+Apr 7th, 2020
+=============
+* Version `0.30.0` of the client extension
+    * PG12: Use postgis_raster instead of custom geomval.
+* Version `0.39.0` of the server extension
+    * PG12: Use postgis_raster instead of custom geomval.
+
+Mar 17th, 2020
+==============
+* Version `0.29.0` of the client extension
+    * Adapted client to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Version `0.38.0` of the server extension
+    * Adapted server to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Update dependencies to fix vulnerabilities
+
 Mar 6th, 2020
 ==============
 * Version `0.23.1` of the Python library

client/.gitignore

@@ -9,3 +9,4 @@ cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
 cdb_geocoder_client--0.3.0.sql
+cdb_dataservices_client.control

client/Makefile

@@ -1,27 +1,30 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_client
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.30.0
 
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
 # PG12 compatibility
 PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
 PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
 PLPYTHONU := plpythonu
+POSTGIS := postgis
 ifeq ($(PG_12_GE), true)
 PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
 endif
 
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 # OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # DATA =  $(NEW_EXTENSION_ARTIFACT) \
 #   $(OLD_VERSIONS) \
@@ -35,7 +38,7 @@ SOURCES_DATA_DIR = sql/
 
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
-TEST_DIR = test_out
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -51,7 +54,6 @@ TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb)
 GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES))
 
 # postgres build stuff
-PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
@@ -62,29 +64,21 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR)/*.sql | \
-	$(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
+	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
-	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
 	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
 	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
@@ -95,37 +89,30 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 	rm -f $(GENERATED_SQL_FILES)
-	rm -rf $(TEST_DIR)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
 
-# Replacing variables defined within test files
-replace_variables:
-	mkdir -p $(TEST_DIR)
-	mkdir -p $(TEST_DIR)/expected
-	mkdir -p $(TEST_DIR)/sql
-	for f in $(REGRESS); do \
-		cat test/sql/$${f}.sql | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		>> $(TEST_DIR)/sql/$${f}.sql; \
-	done
-	for f in $(REGRESS_EXPEC); do \
-		cat test/expected/$${f}.out | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		>> $(TEST_DIR)/expected/$${f}.out; \
-	done
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-install: replace_variables deploy
+.PHONY: install
+install: deploy
+	@echo "Install overriden"

client/README.md

@@ -37,7 +37,7 @@ psql -U postgres cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db
 and then:
 
 ```sql
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
 ```
 
 The extension creation in the user's db requires **superuser** privileges.

client/carto-package.json

@@ -4,7 +4,7 @@
     "requires": {
       "postgresql": "^11.0.0",
       "postgis": "^2.5.0.0",
-      "carto_postgresql_ext": "^0.32.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
     }

client/cdb_dataservices_client--0.29.0--0.30.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.30.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- In 0.30.0 we removed cdb_dataservices_client.geomval and rely on postgis_raster if necessary
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/cdb_dataservices_client--0.30.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- Create cdb_dataservices_client.geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/cdb_dataservices_client.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.29.0'
-requires = 'plproxy, cartodb'
+default_version = '@@EXTVERSION@@'
+requires = 'plproxy, cartodb, @@plpythonu@@, @@postgis@@'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.28.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/old_versions/cdb_dataservices_client--0.29.0--0.28.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/sql/05_utils.sql

@@ -18,16 +18,3 @@ $func$ LANGUAGE plpgsql;
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
     SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
 $f$ LANGUAGE sql IMMUTABLE;
-
-
--- PG12_DEPRECATED
--- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
-DO $$
-BEGIN
-    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
-        CREATE TYPE cdb_dataservices_client.geomval AS (
-            geom geometry,
-            val double precision
-        );
-    END IF;
-END$$;

client/test/expected/00_installation_test.out

@@ -1,30 +1,17 @@
 -- Only show warning or error messages in the tests output
-SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
--- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
--- Mock the server connection to point to this very test db
-SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');
+\set ECHO none
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the user configuration
-SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}');
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the server schema
-CREATE SCHEMA cdb_dataservices_server;
--- Create a test user to check permissions
-DROP ROLE IF EXISTS test_regular_user;
-CREATE ROLE test_regular_user;
-GRANT publicuser TO test_regular_user;
-ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_dataservices_client;
+CREATE SCHEMA
+DROP ROLE
+CREATE ROLE
+GRANT ROLE
+ALTER ROLE

client/test/sql/00_installation_test.sql

@@ -1,13 +1,13 @@
 -- Only show warning or error messages in the tests output
+\set ECHO none
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
+
+\unset ECHO
+\unset QUIET
 
 -- Mock the server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');

docs/developer-center/reference/07-demographic-functions.md

@@ -1,5 +1,9 @@
 ### Demographic Functions
 
+<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
+  <p>We will no longer support these demographic functions from the 1st of January 2021. If you want to collect demographic reports around a point location, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
+</div>
+
 The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions]({{site.dataservicesapi_docs}}/reference/#geocoding-functions) documentation.
 
 _**Note:** The Demographic Snapshot functions are only available for the United States._

docs/developer-center/reference/09-segmentation-functions.md

@@ -1,5 +1,9 @@
 ## Segmentation Functions
 
+<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
+  <p>We will no longer support these segmentations functions from the 1st of January 2021. For a similar functionality, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
+</div>
+
 The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions]({{site.dataservicesapi_docs}}/reference/#geocoding-functions) documentation.
 
 _**Note:** The Segmentation Snapshot functions are only available for the United States. Our first release (May 18, 2016) is derived from Census 2010 variables. Our next release will be based on Census 2014 data. For the latest information, see the [Open Segments](https://github.com/CartoDB/open-segments) project repository._

server/extension/.gitignore

@@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql
 cdb_geocoder_server--0.2.0.sql
 cdb_geocoder_server--0.3.0.sql
 cdb_geocoder_server--0.4.0.sql
-cdb_geocoder_server--0.5.0.sql
+cdb_geocoder_server--0.5.0.sql
+cdb_dataservices_server.control
+./test/sql/366_empty_table_test.sql

server/extension/Makefile

@@ -1,20 +1,35 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.39.3
+
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
+PG_CONFIG ?= pg_config
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+POSTGIS := postgis
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
+endif
+
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
-TEST_DIR = test_out
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -27,106 +42,72 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 SOURCES_DATA = $(wildcard sql/*.sql)
-SOURCES_DATA_DIR_OUT = sql_out/
 
 # postgres build stuff
-PG_CONFIG = pg_config
+
+# The targets listed under REGRESS_PREP are executed before regress
+PYTHON_TESTS = ./test/sql/366_empty_table_test.sql
+REGRESS_PREP = $(PYTHON_TESTS)
+
+./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template
+	$(SED) $(REPLACEMENTS) $@.template > $@
+
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-# PG12 compatibility
-PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
-PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
-PLPYTHONU := plpythonu
-ifeq ($(PG_12_GE), true)
-PLPYTHONU := plpython3u
-endif
-
-REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
-ifneq ($(NEW_VERSION),)
-REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-endif
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
-
-$(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) $(REPLACEMENTS)  $< > $@
+	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
 .PHONY: all
-all: replace_variables $(DATA)
+all: $(DATA)
 
 .PHONY: release
-release: replace_variables $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
-	git add $(EXTENSION).control
-	cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
-	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
-	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
+	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql.in and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
+	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql.in and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
 	@echo
 
 # Only meant for development time, do not use once a version is released
 .PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
-	rm -rf $(TEST_DIR)
-	rm -rf $(SOURCES_DATA_DIR_OUT)
-	rm -rf $(EXTENSION).control
+	rm -f $(PYTHON_TESTS)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
+	
+
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
-
-# Replacing variables defined within test files
-replace_variables:
-	# tests
-	mkdir -p $(TEST_DIR)
-	mkdir -p $(TEST_DIR)/expected
-	mkdir -p $(TEST_DIR)/sql
-	for f in $(REGRESS); do \
-		cat test/sql/$${f}.sql | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(TEST_DIR)/sql/$${f}.sql; \
-	done
-	for f in $(REGRESS_EXPEC); do \
-		cat test/expected/$${f}.out | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(TEST_DIR)/expected/$${f}.out; \
-	done
-	# data
-	mkdir -p $(SOURCES_DATA_DIR_OUT)
-	for f in $(notdir $(SOURCES_DATA)); do \
-		cat $(SOURCES_DATA_DIR)/$${f} | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(SOURCES_DATA_DIR_OUT)/$${f}; \
-	done
 
+.PHONY: install
 install: deploy
-
-reinstall: install
-	psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"
+	@echo "Install overriden"

server/extension/carto-package.json

@@ -4,11 +4,11 @@
     "requires": {
       "postgresql": "^10.0.0",
       "postgis": "^2.4.0.0",
-      "carto_postgresql_ext": "^0.23.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
-      "dataservices-api-server-python-lib": "^0.19.1",
+      "dataservices-api-server-python-lib": "^0.23.1",
       "observatory-server-extension": "^1.9.0"
     }
   }
-}
+}

server/extension/cdb_dataservices_server--0.39.2--0.39.3.sql

@@ -0,0 +1,42 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.3'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.tomtom import TomTomGeocoder
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using TomTom')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.39.3--0.39.2.sql

@@ -0,0 +1,50 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.tomtom import TomTomGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using TomTom')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.38.0'
-requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
+default_version = '@@EXTVERSION@@'
+requires = '@@plpythonu@@, plproxy, @@postgis@@, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.37.0--0.38.0.sql

@@ -0,0 +1,76 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.38.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_server.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.38.0--0.37.0.sql

@@ -0,0 +1,71 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_server.geomval RESTRICT;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.38.0--0.39.0.sql

@@ -0,0 +1,11 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+-- In 0.39.0 we removed cdb_dataservices_client.geomval and rely on postgis_raster if necessary
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_server.geomval RESTRICT;
+END$$;

server/extension/old_versions/cdb_dataservices_server--0.39.0--0.38.0.sql

@@ -1,4 +1,9 @@
--- PG12_DEPRECATED
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.38.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
 -- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
 DO $$
 BEGIN
@@ -8,4 +13,4 @@ BEGIN
             val double precision
         );
     END IF;
-END$$;
+END$$;

server/extension/old_versions/cdb_dataservices_server--0.39.0--0.39.1.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.39.1--0.39.0.sql

@@ -0,0 +1,76 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.39.1--0.39.2.sql

@@ -0,0 +1,140 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing: ' + str(e))
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
+
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing: ' + str(e))
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.39.2--0.39.1.sql

@@ -0,0 +1,139 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
+
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/100_routing_helper.sql

@@ -66,11 +66,12 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     import sys
     service_manager.quota_service.increment_failed_service_use()
     service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to calculate Mapbox routing')
+    raise Exception('Error trying to calculate Mapbox routing: ' + str(e))
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
   username TEXT,
   orgname TEXT,
@@ -134,7 +135,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     import sys
     service_manager.quota_service.increment_failed_service_use()
     service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to calculate TomTom routing')
+    raise Exception('Error trying to calculate TomTom routing: ' + str(e))
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/20_geocode_street.sql

@@ -275,10 +275,8 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
-  from iso3166 import countries
   from cartodb_services.tools import ServiceManager, QuotaExceededException
   from cartodb_services.tomtom import TomTomGeocoder
-  from cartodb_services.tools.country import country_to_iso3
   from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
 
   import cartodb_services
@@ -290,15 +288,9 @@ RETURNS Geometry AS $$
     service_manager.assert_within_limits()
     geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
 
-    country_iso3166 = None
-    if country:
-      country_iso3 = country_to_iso3(country)
-      if country_iso3:
-        country_iso3166 = countries.get(country_iso3).alpha2.lower()
-
     coordinates = geocoder.geocode(searchtext=searchtext, city=city,
                                    state_province=state_province,
-                                   country=country_iso3166)
+                                   country=country)
     if coordinates:
       service_manager.quota_service.increment_success_service_use()
       plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])

server/extension/sql/85_isodistance.sql

@@ -11,9 +11,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  if user_isolines_config.google_services_user:
-    raise Exception('This service is not available for google service users.')
-
   params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
 
   with metrics('cdb_isodistance', user_isolines_config, logger, params):

server/extension/sql/90_isochrone.sql

@@ -11,9 +11,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  if user_isolines_config.google_services_user:
-    raise Exception('This service is not available for google service users.')
-
   params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
 
   with metrics('cdb_isochrone', user_isolines_config, logger, params):

server/extension/test/expected/00_install_test.out

@@ -1,13 +1,9 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
  cdb_conf_setconf 
@@ -71,6 +67,7 @@ BEGIN
 END
 $$
 LANGUAGE plpgsql;
+CREATE FUNCTION
 -- Set user quota
 SELECT cartodb.CDB_SetUserQuotaInBytes(0);
  cdb_setuserquotainbytes 

server/extension/test/sql/00_install_test.sql

@@ -1,14 +1,10 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -18,7 +18,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status'
 class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
     MAX_BATCH_SIZE = 1000000  # From the docs
     MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
-    BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
+    BATCH_URL = 'https://batch.geocoder.api.here.com/6.2/jobs'
     # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
     META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
     MAX_STALLED_RETRIES = 100
@@ -126,7 +126,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
             status=polling_root.find('./Response/Status').text)
 
     def _download_results(self, job_id):
-        result_r = self.session.get("{}/{}/result".format(self.BATCH_URL, job_id),
+        result_r = self.session.get("{}/{}/all".format(self.BATCH_URL, job_id),
                                     params=self.credentials_params,
                                     timeout=(self.connect_timeout, self.read_timeout))
         root_zip = zipfile.ZipFile(io.BytesIO(result_r.content))
@@ -147,6 +147,9 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
                                             precision,
                                             [match_type] if match_type else []
                                         )))
-
+                    elif row['matchLevel'] == 'NOMATCH':
+                        results.append((row['recId'], [], {}))
+                    elif row['matchLevel'] == 'FAILED':
+                        results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
         return results
 

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -115,7 +115,7 @@ class HereMapsRoutingIsoline(Traceable):
 
     def __parse_source_param(self, source, options):
         key = 'start'
-        if 'is_destination' in options and options['is_destination']:
+        if 'is_destination' in options and options['is_destination'].lower() == 'true':
             key = 'destination'
 
         return {key: source}

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -105,7 +105,7 @@ class MapboxIsolines():
             elif response.status_code == requests.codes.unprocessable_entity:
                 return []
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/mapbox/routing.py

@@ -94,7 +94,7 @@ class MapboxRouting(Traceable):
             elif response.status_code == requests.codes.unprocessable_entity:
                 return MapboxRoutingResponse(None, None, None)
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py

@@ -10,6 +10,7 @@ from cartodb_services.metrics import Traceable
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools.normalize import normalize
+from cartodb_services.tools.country import country_to_iso3
 
 HOST = 'https://api.tomtom.com'
 API_BASEURI = '/search/2'
@@ -48,7 +49,7 @@ class TomTomGeocoder(Traceable):
     def _request_uri(self, searchtext, country=None, apiKey=None):
         baseuri = REQUEST_BASEURI
         if country:
-            baseuri += '&countrySet={}'.format(country)
+            baseuri += '&countrySet={}'.format(country_to_iso3(country) or country)
         baseuri = baseuri + '&key={apiKey}' if apiKey else baseuri
         return URITemplate(baseuri).expand(apiKey=apiKey,
                                            searchtext=searchtext.encode('utf-8'))

server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py

@@ -82,7 +82,7 @@ class TomTomIsolines():
             elif response.status_code == requests.codes.unprocessable_entity:
                 return []
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py

@@ -120,7 +120,7 @@ class TomTomRouting(Traceable):
             elif response.status_code == requests.codes.unprocessable_entity:
                 return TomTomRoutingResponse(None, None, None)
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/requirements.txt

@@ -4,8 +4,11 @@ python-dateutil==2.2
 googlemaps==2.5.1
 rollbar==0.13.2
 # Dependency for googlemaps package
-requests==2.9.1
+requests==2.20.0
 rratelimit==0.0.4
+# botocore 1.19.0 introduced an error `ImportError: cannot import name 'IPV6_ADDRZ_RE'`
+# and mapbox depends on botocore<=1.19.0 so forcing 1.18.18
+botocore==1.18.18
 mapbox==0.14.0
 pygeocodio==0.11.1
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.23.1',
+    version='0.23.5',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_heremapsrouting.py

@@ -204,6 +204,16 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
                          'fastest;car;traffic:false;motorway:-1')
 
     def test_source_parameters_works_properly(self, req_mock):
+        req_mock.register_uri('GET', requests_mock.ANY,
+                              text=self.GOOD_RESPONSE)
+        response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',
+                                                    ['1000', '2000'],
+                                                    ['is_destination=false'])
+        parsed_url = urlparse(req_mock.request_history[0].url)
+        url_params = parse_qs(parsed_url.query)
+        self.assertEqual(url_params['start'][0], 'geo!33.0,1.0')
+
+    def test_destination_parameters_works_properly(self, req_mock):
         req_mock.register_uri('GET', requests_mock.ANY,
                               text=self.GOOD_RESPONSE)
         response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',

test/integration/test_street_functions.py

@@ -280,18 +280,18 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
     def test_templating_geocoding(self):
         query = "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \
                 "cdb_dataservices_client.cdb_bulk_geocode_street_point(" \
-                "'select 1 as cartodb_id, ''Logroño'' as city', " \
+                "'select 1 as cartodb_id, ''Valladolid'' as city', " \
                 "'city || '', '' || ''Spain''') " \
                 "UNION " \
                 "SELECT cartodb_id, st_x(the_geom), st_y(the_geom) from " \
                 "cdb_dataservices_client.cdb_bulk_geocode_street_point(" \
-                "'select 2 as cartodb_id, ''Logroño'' as city', " \
-                "'city || '', '' || ''Argentina''')"
+                "'select 2 as cartodb_id, ''Valladolid'' as city', " \
+                "'city || '', '' || ''Mexico''')"
         response = self._run_authenticated(query)
 
         points_by_cartodb_id = {
-            1: self.fixture_points['Logroño, Spain'],
-            2: self.fixture_points['Logroño, Argentina']
+            1: self.fixture_points['Valladolid, Spain'],
+            2: self.fixture_points['Valladolid, Mexico']
         }
         self.assert_close_points(self._x_y_by_cartodb_id(response), points_by_cartodb_id)
 

test/requirements.txt

@@ -1,3 +1,3 @@
 # Integration tests dependencies
-requests==2.9.1
+requests==2.20.0
 nose==1.3.7