Merge pull request #587 from CartoDB/development

python-0.23.1 deploy to production

.github/workflows/main.yml

@@ -0,0 +1,56 @@
+name: dataservices-api PR testing
+on: push
+jobs:
+  dataservices-api:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        pg_version: [10, 12]
+    env:
+        PG_VERSION: ${{ matrix.pg_version }}
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup gcloud authentication
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCS }}
+
+      - name: Pull base image
+        run: gcloud auth configure-docker && docker pull gcr.io/cartodb-on-gcp-ci-testing/cartodb-postgresql-base:${{ matrix.pg_version }}
+
+      - name: Checkout ci tools repository
+        uses: actions/checkout@v2
+        with:
+          repository: CartoDB/ci-tools
+          path: ci-tools
+          token: ${{ secrets.CARTOFANTE_PAT }}
+
+      - name: Copy ci files to root
+        run: cp ci-tools/repos/${{ github.event.repository.name }}/* .
+
+      - name: Start docker-compose services
+        run: docker-compose -f docker-compose.yaml up -d
+
+      - name: Install required python3 libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services"
+        if: env.PG_VERSION == 12
+
+      - name: Install required python libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services"
+
+      - name: Run python library tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test"
+        env:
+           MAPBOX_API_KEY: ${{ secrets.MAPBOX_API_KEY }}
+           TOMTOM_API_KEY: ${{ secrets.TOMTOM_API_KEY }}
+           GEOCODIO_API_KEY: ${{ secrets.GEOCODIO_API_KEY }}
+        timeout-minutes: 5
+
+      - name: Run server tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)"
+        timeout-minutes: 5
+
+      - name: Run client tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)"
+        timeout-minutes: 5

NEWS.md

@@ -1,3 +1,15 @@
+Mar 6th, 2020
+==============
+* Version `0.23.1` of the Python library
+    * Added query per second throttle to TomTom bulk API calls
+
+Feb 12th, 2020
+==============
+* Version `0.23.0` of the Python library
+    * Added Mapbox true isolines
+* Version `0.37.0` of the server extension
+    * Added Mapbox true isolines
+
 Jan 20th, 2020
 ==============
 * Version `0.22.0` of the Python library

README.md

@@ -40,6 +40,7 @@ Steps to deploy a new Data Services API version :
     # in dataservices-api repo root path:
     cd server/lib/python/cartodb_services && sudo pip install -r requirements.txt && sudo pip install . --upgrade
     ```
+    **CLOUD DEPLOY NOTE**: we were not installing automatically `requirements.txt`, so we fixed it in https://github.com/CartoDB/cartodb-platform/pull/6187 . Please, be aware that in some corner cases scenarios, rolling back to a previous version might require to manually force-install some dependency versions that were upgraded previously in this step.
 
 - Create a database to hold all the server part and a user for it
 

client/Makefile

@@ -14,6 +14,14 @@ AWK = awk
 PG_CONFIG         = pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
+
 # OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # DATA =  $(NEW_EXTENSION_ARTIFACT) \
 #   $(OLD_VERSIONS) \
@@ -25,9 +33,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 
-
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test_out
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -54,7 +62,8 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR)/*.sql >> $@
+	cat $(SOURCES_DATA_DIR)/*.sql | \
+	$(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
 ifeq ($(PG_PARALLEL), 0)
 # Remove PARALLEL in aggregates and functions
 	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
@@ -86,6 +95,7 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 	rm -f $(GENERATED_SQL_FILES)
+	rm -rf $(TEST_DIR)
 
 # If needed remove PARALLEL tags from the release files
 release_remove_parallel_deploy:
@@ -97,9 +107,25 @@ ifeq ($(PG_PARALLEL), 0)
 	done
 endif
 
+# Replacing variables defined within test files
+replace_variables:
+	mkdir -p $(TEST_DIR)
+	mkdir -p $(TEST_DIR)/expected
+	mkdir -p $(TEST_DIR)/sql
+	for f in $(REGRESS); do \
+		cat test/sql/$${f}.sql | \
+		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
+		>> $(TEST_DIR)/sql/$${f}.sql; \
+	done
+	for f in $(REGRESS_EXPEC); do \
+		cat test/expected/$${f}.out | \
+		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
+		>> $(TEST_DIR)/expected/$${f}.out; \
+	done
+
 # Install the current release into the PostgreSQL extensions directory
 deploy: release_remove_parallel_deploy
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-install: deploy
+install: replace_variables deploy

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.28.0'
+default_version = '0.29.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/sql/05_utils.sql

@@ -17,4 +17,17 @@ $func$ LANGUAGE plpgsql;
 -- Taken from https://stackoverflow.com/a/48013356/351721
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
     SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
-$f$ LANGUAGE sql IMMUTABLE;
+$f$ LANGUAGE sql IMMUTABLE;
+
+
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/sql/20_table_augmentation.sql

@@ -138,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeas
         )
 
     return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
     username text,
@@ -231,7 +231,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMea
             fdw_server=plpy.quote_literal(server_name)))
 
     return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
     username text,

client/test/expected/00_installation_test.out

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;
 -- Install the extension

client/test/sql/00_installation_test.sql

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION plproxy;
 

server/extension/Makefile

@@ -1,11 +1,10 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
 # The new version to be generated from templates
 SED = sed
 ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
 NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
 AWK = awk
 
@@ -14,7 +13,8 @@ PG_CONFIG         = pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test_out
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -27,15 +27,29 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 SOURCES_DATA = $(wildcard sql/*.sql)
+SOURCES_DATA_DIR_OUT = sql_out/
 
 # postgres build stuff
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+endif
+
+REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
+ifneq ($(NEW_VERSION),)
+REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
+endif
+
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR)/*.sql >> $@
+	cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@
 ifeq ($(PG_PARALLEL), 0)
 # Remove PARALLEL in aggregates and functions
 	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
@@ -44,16 +58,19 @@ ifeq ($(PG_PARALLEL), 0)
 	mv $(TMPFILE) $@
 endif
 
+$(EXTENSION).control: $(EXTENSION).control.in Makefile
+	$(SED) $(REPLACEMENTS)  $< > $@
+
 .PHONY: all
-all: $(DATA)
+all: replace_variables $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: replace_variables $(EXTENSION).control $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 	git add $(EXTENSION).control
-	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
@@ -66,6 +83,9 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 .PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
+	rm -rf $(TEST_DIR)
+	rm -rf $(SOURCES_DATA_DIR_OUT)
+	rm -rf $(EXTENSION).control
 
 # If needed remove PARALLEL tags from the release files
 release_remove_parallel_deploy:
@@ -82,6 +102,30 @@ deploy: release_remove_parallel_deploy
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
+# Replacing variables defined within test files
+replace_variables:
+	# tests
+	mkdir -p $(TEST_DIR)
+	mkdir -p $(TEST_DIR)/expected
+	mkdir -p $(TEST_DIR)/sql
+	for f in $(REGRESS); do \
+		cat test/sql/$${f}.sql | \
+		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
+		> $(TEST_DIR)/sql/$${f}.sql; \
+	done
+	for f in $(REGRESS_EXPEC); do \
+		cat test/expected/$${f}.out | \
+		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
+		> $(TEST_DIR)/expected/$${f}.out; \
+	done
+	# data
+	mkdir -p $(SOURCES_DATA_DIR_OUT)
+	for f in $(notdir $(SOURCES_DATA)); do \
+		cat $(SOURCES_DATA_DIR)/$${f} | \
+		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
+		> $(SOURCES_DATA_DIR_OUT)/$${f}; \
+	done
+
 install: deploy
 
 reinstall: install

server/extension/cdb_dataservices_server--0.36.0--0.37.0.sql

@@ -0,0 +1,126 @@
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.37.0--0.36.0.sql

@@ -0,0 +1,129 @@
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.36.0'
+default_version = '0.38.0'
-requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
+requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/05_postgis3_adapter.sql

@@ -0,0 +1,11 @@
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_server.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

server/extension/sql/100_routing_helper.sql

@@ -69,7 +69,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate Mapbox routing')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
   username TEXT,
@@ -137,7 +137,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate TomTom routing')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
   username TEXT,
@@ -198,4 +198,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     raise Exception('Error trying to calculate mapzen routing')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/105_route_between_points.sql

@@ -37,7 +37,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
@@ -76,4 +76,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/10_redis_helper.sql

@@ -15,4 +15,4 @@ RETURNS boolean AS $$
       'redis_metrics_connection': redis_metrics_connection,
     }
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/110_data_observatory_augmentation.sql

@@ -14,7 +14,7 @@ RETURNS text AS $$
   user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   return user_obs_config.connection_str
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
   username TEXT,
@@ -68,7 +68,7 @@ RETURNS json AS $$
         raise Exception('Error trying to obs_get_demographic_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
   username TEXT,
@@ -125,7 +125,7 @@ RETURNS SETOF JSON AS $$
         raise Exception('Error trying to obs_get_demographic_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
   username TEXT,
@@ -177,7 +177,7 @@ RETURNS json AS $$
         raise Exception('Error trying to obs_get_segment_snapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
   username TEXT,
@@ -232,7 +232,7 @@ RETURNS SETOF JSON AS $$
         raise Exception('Error trying to OBS_GetSegmentSnapshot')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
   username TEXT,
@@ -289,7 +289,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetMeasure')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
   username TEXT,
@@ -344,7 +344,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetCategory')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
   username TEXT,
@@ -401,7 +401,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetUSCensusMeasure')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
   username TEXT,
@@ -456,7 +456,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetUSCensusCategory')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
   username TEXT,
@@ -511,7 +511,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetPopulation')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
   username TEXT,
@@ -566,7 +566,7 @@ RETURNS NUMERIC AS $$
         raise Exception('Error trying to OBS_GetMeasureById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
   username TEXT,
@@ -627,7 +627,7 @@ RETURNS TABLE (
         raise Exception('Error trying to OBS_GetData')
     finally:
         quota_service.increment_total_service_use(len(geomvals))
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
   username TEXT,
@@ -688,7 +688,7 @@ RETURNS TABLE (
         raise Exception('Error trying to OBS_GetData')
     finally:
         quota_service.increment_total_service_use(len(geomrefs))
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
   username TEXT,
@@ -736,7 +736,7 @@ RETURNS JSON AS $$
         import sys
         logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
         raise Exception('Error trying to OBS_GetMeta')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_MetadataValidation(
   username TEXT,
@@ -782,4 +782,4 @@ RETURNS TABLE(valid boolean, errors text[]) AS $$
         import sys
         logger.error('Error trying to OBS_MetadataValidation', sys.exc_info(), data={"username": username, "orgname": orgname})
         raise Exception('Error trying to OBS_MetadataValidation')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/115_data_observatory_exploration.sql

@@ -55,7 +55,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
         raise Exception('Error trying to OBS_Search')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
   username TEXT,
@@ -113,4 +113,4 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
         raise Exception('Error trying to OBS_GetMeasureById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/120_data_observatory_geometries.sql

@@ -49,7 +49,7 @@ RETURNS geometry(Geometry, 4326) AS $$
         raise Exception('Error trying to OBS_GetBoundary')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
   username TEXT,
@@ -102,7 +102,7 @@ RETURNS TEXT AS $$
         raise Exception('Error trying to OBS_GetBoundaryId')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
   username TEXT,
@@ -155,7 +155,7 @@ RETURNS geometry(Geometry, 4326) AS $$
         raise Exception('Error trying to OBS_GetBoundaryById')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
   username TEXT,
@@ -215,7 +215,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetBoundariesByGeometry')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
   username TEXT,
@@ -277,7 +277,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
   username TEXT,
@@ -337,7 +337,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetPointsByGeometry')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
   username TEXT,
@@ -399,4 +399,4 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
         raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
     finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/125_data_observatory_table_augment.sql

@@ -8,7 +8,7 @@ RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
     return plpy.execute("SELECT * FROM cdb_dataservices_server.__DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
         .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
         )[0]
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.__DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
 RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$

server/extension/sql/15_config_helper.sql

@@ -8,7 +8,7 @@ RETURNS boolean AS $$
     logger_config = LoggerConfig(plpy)
     GD[cache_key] = logger_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 -- This is done in order to avoid an undesired depedency on cartodb extension
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_conf_getconf(input_key text)
@@ -46,7 +46,7 @@ RETURNS boolean AS $$
     geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
     GD[cache_key] = geocoder_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -60,7 +60,7 @@ RETURNS boolean AS $$
     geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = geocoder_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -74,7 +74,7 @@ RETURNS boolean AS $$
     isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = isolines_routing_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -88,7 +88,7 @@ RETURNS boolean AS $$
     routing_config = RoutingConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = routing_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
 RETURNS boolean AS $$
@@ -102,4 +102,4 @@ RETURNS boolean AS $$
     obs_config = ObservatoryConfig(redis_conn, plpy, username, orgname)
     GD[cache_key] = obs_config
     return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/200_quotas.sql

@@ -106,7 +106,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
   ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
 
   return ret
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
@@ -136,7 +136,7 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
       ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
 
   return ret
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
   username TEXT,

server/extension/sql/20_geocode_street.sql

@@ -36,7 +36,7 @@ RETURNS Geometry AS $$
     else:
       raise Exception('Requested geocoder is not available')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
@@ -52,7 +52,7 @@ RETURNS Geometry AS $$
   else:
     raise Exception('Here geocoder is not available for your account.')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -67,7 +67,7 @@ RETURNS Geometry AS $$
   else:
     raise Exception('Google geocoder is not available for your account.')
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -80,7 +80,7 @@ RETURNS Geometry AS $$
   mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -93,7 +93,7 @@ RETURNS Geometry AS $$
   mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -106,7 +106,7 @@ RETURNS Geometry AS $$
   tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -119,7 +119,7 @@ RETURNS Geometry AS $$
   geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
   return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -152,7 +152,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using here maps')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -184,7 +184,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using google maps')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -225,7 +225,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using mapzen')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -271,7 +271,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using Mapbox')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -317,7 +317,7 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using TomTom')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -363,4 +363,4 @@ RETURNS Geometry AS $$
     raise Exception('Error trying to geocode street point using Geocodio')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/210_rates.sql

@@ -16,7 +16,7 @@ RETURNS JSON AS $$
       return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
   else:
       return None
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
   username TEXT,
@@ -40,7 +40,7 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_user_rate_limits(config)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
   username TEXT,
@@ -64,7 +64,7 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_org_rate_limits(config)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
   username TEXT,
@@ -88,4 +88,4 @@ RETURNS VOID AS $$
       period = None
   config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
   config_setter.set_server_rate_limits(config)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;

server/extension/sql/21_bulk_geocode_street.sql

@@ -40,7 +40,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
     plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
     return plpy.execute(plan, [username, orgname, searches])
 
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -51,7 +51,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
   geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -62,7 +62,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
   geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -79,7 +79,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
   geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -96,7 +96,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
   geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
@@ -113,4 +113,4 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
   geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/30_admin0.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode admin0 polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 
 --------------------------------------------------------------------------------

server/extension/sql/40_admin1.sql

@@ -36,7 +36,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode admin1 polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
@@ -74,7 +74,7 @@ RETURNS Geometry AS $$
         raise Exception('Error trying to geocode admin1 polygon')
       finally:
         quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/50_namedplaces.sql

@@ -1,7 +1,7 @@
 ---- cdb_geocode_namedplace_point(city_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -16,12 +16,12 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_namedplace_point(city_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -36,12 +36,12 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -56,7 +56,7 @@ RETURNS Geometry AS $$
     logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
 RETURNS Geometry AS $$
@@ -101,7 +101,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode namedplace point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/60_postalcodes.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision)
 RETURNS Geometry AS $$
@@ -77,7 +77,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code point')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code double precision, country text)
 RETURNS Geometry AS $$
@@ -119,7 +119,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision)
 RETURNS Geometry AS $$
@@ -162,7 +162,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code double precision, country text)
 RETURNS Geometry AS $$

server/extension/sql/70_ips.sql

@@ -35,7 +35,7 @@ RETURNS Geometry AS $$
       raise Exception('Error trying to geocode postal code polygon')
     finally:
       quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 --------------------------------------------------------------------------------
 

server/extension/sql/80_isolines_helper.sql

@@ -55,7 +55,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isolines')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
    username TEXT,
@@ -121,7 +121,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isolines')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
    username TEXT,
@@ -132,7 +132,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
    options text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
   from cartodb_services.tools import ServiceManager
-  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox import MapboxIsolines
   from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
   from cartodb_services.tools import Coordinate
   from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
@@ -144,8 +144,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   service_manager.assert_within_limits()
 
   try:
-    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
-    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
 
     if source:
       lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -169,7 +168,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
         # -- TODO encapsulate this block into a func/method
         locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
         wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
-        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
         multipolygon = plpy.execute(sql, 1)[0]['geom']
       else:
         multipolygon = None
@@ -186,7 +185,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get Mapbox isolines')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
    username TEXT,
@@ -250,7 +249,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get TomTom isolines')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
    username TEXT,
@@ -311,7 +310,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get mapzen isochrones')
   finally:
     quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
    username TEXT,
@@ -322,7 +321,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
    options text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
   from cartodb_services.tools import ServiceManager
-  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox import MapboxIsolines
   from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
   from cartodb_services.tools import Coordinate
   from cartodb_services.tools.coordinates import coordinates_to_polygon
@@ -335,8 +334,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   service_manager.assert_within_limits()
 
   try:
-    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
-    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
 
     if source:
       lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -370,7 +368,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get Mapbox isochrones')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
    username TEXT,
@@ -428,4 +426,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     raise Exception('Error trying to get TomTom isochrones')
   finally:
     service_manager.quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/85_isodistance.sql

@@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- heremaps isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapzen isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -60,7 +60,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapbox isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -74,7 +74,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- tomtom isodistance
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -88,4 +88,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/90_isochrone.sql

@@ -31,7 +31,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- heremaps isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -46,7 +46,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
 
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapzen isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -59,7 +59,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- mapbox isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapbox_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -72,7 +72,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 
 -- tomtom isochrone
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
@@ -85,4 +85,4 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
   result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
   return result
-$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/test/expected/00_install_test.out

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
@@ -39,7 +39,7 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
  cdb_conf_setconf 
 ------------------
  

server/extension/test/expected/100_data_observatory_test.out

@@ -1,3 +1,4 @@
+\set ECHO none
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/00_install_test.sql

@@ -2,7 +2,7 @@
 SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
+CREATE EXTENSION @@plpythonu@@;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
@@ -16,7 +16,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
-SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

server/extension/test/sql/100_data_observatory_test.sql

@@ -1,3 +1,10 @@
+\set ECHO none
+-- add the schema cdb_dataservices_server to the SEARCH_PATH
+DO $$ BEGIN
+    PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
+END $$;
+\set ECHO all
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/366_empty_table_test.sql

@@ -2,6 +2,11 @@
 \set VERBOSITY verbose
 SET client_min_messages TO error;
 
+-- add the schema cdb_dataservices_server to the SEARCH_PATH
+DO $$ BEGIN
+    PERFORM set_config('search_path', current_setting('search_path')||', cdb_dataservices_server', false);
+END $$;
+
 -- Set configuration for a user 'foo'
 DO $$
   import json
@@ -14,7 +19,7 @@ DO $$
   service_config.user.set('soft_obs_general_limit', True)
   service_config.user.set('period_end_date', '20170516')
 
-$$ LANGUAGE plpythonu;
+$$ LANGUAGE @@plpythonu@@;
 
 
 -- Mock Observatory backend function
@@ -37,7 +42,7 @@ GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api;
 SELECT * FROM cdb_dataservices_server.OBS_GetData(
     'foo',
     NULL,
-    '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::public._geomval,
+    '{"(0103000020E61000000100000005000000010000E0F67F52C096D88AE6B25F4440010000E0238052C0BF6D8A1A8D5D4440010000D0DA7E52C05F03F3CC265D444001000020F47E52C0F2DD78AB5D5F4440010000E0F67F52C096D88AE6B25F4440,1)"}'::_geomval,
     '[{"id": 1, "score": 52.7515548093083898758340051256007949661290516400338, "geom_id": "us.census.tiger.census_tract", "denom_id": "us.census.acs.B01003001", "numer_id": "us.census.acs.B03002003", "geom_name": "US Census Tracts", "geom_type": "Geometry", "num_geoms": 2.86483076549783307739486952736, "denom_name": "Total Population", "denom_type": "Numeric", "numer_name": "White Population", "numer_type": "Numeric", "score_rank": 1, "target_area": 0.000307374806576033, "geom_colname": "the_geom", "score_rownum": 1, "target_geoms": null, "denom_colname": "total_pop", "denom_reltype": '
          '"denominator", "geom_timespan": "2015", "normalization": "prenormalized", "numer_colname": "white_pop", "timespan_rank": 1, "geom_tablename": "obs_87a814e485deabe3b12545a537f693d16ca702c2", "max_score_rank": null, "numer_timespan": "2010 - 2014", "suggested_name": "white_pop_2010_2014", "denom_aggregate": "sum", "denom_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "numer_aggregate": "sum", "numer_tablename": "obs_b393b5b88c6adda634b2071a8005b03c551b609a", "timespan_rownum": 1, "geom_description": "Census tracts are small, relatively permanent statistical subdivisions of a county or equivalent entity that are updated by local participants prior to each decennial census as part of the Census Bureau’s Participant Statistical Areas Program. The Census Bureau delineates census tracts in situations where no local participant existed or where state, local, or tribal governments'
          'declined to participate. The primary purpose of census tracts is to provide a stable set of geographic units for the presentation of statistical data.\r\n\r\nCensus tracts generally have a population size between 1,200 and 8,000 people, with an optimum size of 4,000 people. A census tract usually covers a contiguous area; however, the spatial size of census tracts varies widely depending on the density of settlement. Census tract boundaries are delineated with the intention of being maintained over a long time so that statistical comparisons can be made from census to census. Census tracts occasionally are split due to population growth or merged as a result of substantial population decline.\r\n\r\nCensus tract boundaries generally follow visible and identifiable features. They may follow nonvisible legal boundaries, such as minor civil division (MCD) or incorporated place boundaries'

server/lib/python/cartodb_services/cartodb_services/__init__.py

@@ -34,4 +34,4 @@ def _reset():
     plpy = None
     GD = None
 
-from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
+from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -1,8 +1,8 @@
-from google import GoogleMapsBulkGeocoder
+from cartodb_services.google import GoogleMapsBulkGeocoder
-from here import HereMapsBulkGeocoder
+from cartodb_services.here import HereMapsBulkGeocoder
-from tomtom import TomTomBulkGeocoder
+from cartodb_services.tomtom import TomTomBulkGeocoder
-from mapbox import MapboxBulkGeocoder
+from cartodb_services.mapbox import MapboxBulkGeocoder
-from geocodio import GeocodioBulkGeocoder
+from cartodb_services.geocodio import GeocodioBulkGeocoder
 
 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
     'google': GoogleMapsBulkGeocoder,

server/lib/python/cartodb_services/cartodb_services/config/__init__.py

@@ -1,3 +1,3 @@
-from service_configuration import ServiceConfiguration
+from .service_configuration import ServiceConfiguration
-from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
-from legacy_rate_limits import RateLimitsConfigLegacyBuilder
+from .legacy_rate_limits import RateLimitsConfigLegacyBuilder

server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py

@@ -1,5 +1,5 @@
 import json
-from rate_limits import RateLimitsConfig
+from cartodb_services.config.rate_limits import RateLimitsConfig
 
 class RateLimitsConfigLegacyBuilder(object):
     """

server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py

@@ -1,6 +1,6 @@
 import json
 
-from service_configuration import ServiceConfiguration
+from cartodb_services.config.service_configuration import ServiceConfiguration
 
 class RateLimitsConfig(object):
     """

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -1,7 +1,7 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
-from tools import QuotaExceededException, Logger
+from cartodb_services.tools import QuotaExceededException, Logger
 from collections import namedtuple
 import json
 
@@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
                 except Exception as e:
                     import sys
                     logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
-                    metadata['processing_error'] = 'Error: {}'.format(e.message)
+                    metadata['processing_error'] = 'Error: {}'.format(e)
                     results.append([result[0], None, json.dumps(metadata)])
                     failed_count += 1
 

server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GeocodioGeocoder
+from cartodb_services.geocodio.geocoder import GeocodioGeocoder
-from bulk_geocoder import GeocodioBulkGeocoder
+from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GoogleMapsGeocoder
+from cartodb_services.google.geocoder import GoogleMapsGeocoder
-from bulk_geocoder import GoogleMapsBulkGeocoder
+from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py

@@ -1,5 +1,5 @@
 from multiprocessing import Pool
-from exceptions import MalformedResult
+from cartodb_services.google.exceptions import MalformedResult
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.geocoder import compose_address, geocoder_error_response
 from cartodb_services.google import GoogleMapsGeocoder

server/lib/python/cartodb_services/cartodb_services/google/client_factory.py

@@ -3,7 +3,7 @@
 
 import googlemaps
 import base64
-from exceptions import InvalidGoogleCredentials
+from cartodb_services.google.exceptions import InvalidGoogleCredentials
 
 
 class GoogleMapsClientFactory():

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,9 +1,11 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
+try:
+    from urlparse import parse_qs
+except:
+    from urllib.parse import parse_qs
 
-from urlparse import parse_qs
+from cartodb_services.google.exceptions import MalformedResult
-
-from exceptions import MalformedResult
 from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
 from cartodb_services.google.exceptions import InvalidGoogleCredentials
 from client_factory import GoogleMapsClientFactory

server/lib/python/cartodb_services/cartodb_services/here/__init__.py

@@ -1,3 +1,3 @@
-from geocoder import HereMapsGeocoder
+from cartodb_services.here.geocoder import HereMapsGeocoder
-from bulk_geocoder import HereMapsBulkGeocoder
+from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
-from routing import HereMapsRoutingIsoline
+from cartodb_services.here.routing import HereMapsRoutingIsoline

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -1,7 +1,7 @@
 import requests
 import json
 
-from exceptions import WrongParams
+from cartodb_services.here.exceptions import WrongParams
 from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 

server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py

@@ -1,5 +1,4 @@
-from routing import MapboxRouting, MapboxRoutingResponse
+from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
-from geocoder import MapboxGeocoder
+from cartodb_services.mapbox.geocoder import MapboxGeocoder
-from bulk_geocoder import MapboxBulkGeocoder
+from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
-from isolines import MapboxIsolines, MapboxIsochronesResponse
+from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse
-from matrix_client import MapboxMatrixClient

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -1,171 +1,142 @@
-'''
-Python implementation for Mapbox services based isolines.
-Uses the Mapbox Time Matrix service.
-'''
-
 import json
+import requests
+from uritemplate import URITemplate
+
+from cartodb_services.tools.exceptions import ServiceException
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools import Coordinate
-from cartodb_services.tools.spherical import (get_angles,
+
-                                              calculate_dest_location)
+BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
-from cartodb_services.mapbox.matrix_client import (validate_profile,
+
-                                                   DEFAULT_PROFILE,
+PROFILE_DRIVING = 'driving'
-                                                   PROFILE_WALKING,
+PROFILE_CYCLING = 'cycling'
-                                                   PROFILE_DRIVING,
+PROFILE_WALKING = 'walking'
-                                                   PROFILE_CYCLING,
+DEFAULT_PROFILE = PROFILE_DRIVING
-                                                   ENTRY_DURATIONS,
+
-                                                   ENTRY_DESTINATIONS,
+MAX_TIME_RANGE = 60 * 60  # The maximum time that can be specified is 60 minutes.
-                                                   ENTRY_LOCATION)
+                          # https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
     PROFILE_CYCLING: 16.67,  # In m/s, assuming 60km/h max speed
-    PROFILE_DRIVING: 41.67  # In m/s, assuming 140km/h max speed
+    PROFILE_DRIVING: 38.89  # In m/s, assuming 140km/h max speed
 }
 
-DEFAULT_NUM_ANGLES = 24
+VALID_PROFILES = (PROFILE_DRIVING,
-DEFAULT_MAX_ITERS = 5
+                  PROFILE_CYCLING,
-DEFAULT_TOLERANCE = 0.1
+                  PROFILE_WALKING)
 
-MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES
+ENTRY_FEATURES = 'features'
-MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS
+ENTRY_GEOMETRY = 'geometry'
-MATRIX_TOLERANCE = DEFAULT_TOLERANCE
+ENTRY_COORDINATES = 'coordinates'
-
-UNIT_FACTOR_ISOCHRONE = 1.0
-UNIT_FACTOR_ISODISTANCE = 1000.0
-DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE
 
 
 class MapboxIsolines():
     '''
-    Python wrapper for Mapbox services based isolines.
+    Python wrapper for Mapbox based isolines.
     '''
 
-    def __init__(self, matrix_client, logger, service_params=None):
+    def __init__(self, apikey, logger, service_params=None):
         service_params = service_params or {}
-        self._matrix_client = matrix_client
+        self._apikey = apikey
         self._logger = logger
 
-    def _calculate_matrix_cost(self, origin, targets, isorange,
+    def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
-                               profile=DEFAULT_PROFILE,
+        uri = URITemplate(BASEURI).expand(apikey=self._apikey,
-                               unit_factor=UNIT_FACTOR_ISOCHRONE,
+                                          coordinates=origin,
-                               number_of_angles=MATRIX_NUM_ANGLES):
+                                          contours_minutes=time_range,
-        response = self._matrix_client.matrix([origin] + targets,
+                                          profile=profile)
-                                              profile)
+        return uri
+
+    def _validate_profile(self, profile):
+        if profile not in VALID_PROFILES:
+            raise ValueError('{profile} is not a valid profile. '
+                             'Valid profiles are: {valid_profiles}'.format(
+                                 profile=profile,
+                                 valid_profiles=', '.join(
+                                     [x for x in VALID_PROFILES])))
+
+    def _validate_time_ranges(self, time_ranges):
+        for time_range in time_ranges:
+            if time_range > MAX_TIME_RANGE:
+                raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format(
+                                    max_time_range=MAX_TIME_RANGE))
+
+    def _parse_coordinates(self, boundary):
+        coordinates = boundary.get(ENTRY_COORDINATES, [])
+        return [Coordinate(c[0], c[1]) for c in coordinates]
+
+    def _parse_isochrone_service(self, response):
         json_response = json.loads(response)
-        if not json_response:
-            return []
 
-        costs = [None] * number_of_angles
+        coordinates = []
-        destinations = [None] * number_of_angles
+        if json_response:
+            for feature in json_response[ENTRY_FEATURES]:
+                geometry = feature[ENTRY_GEOMETRY]
+                coordinates.append(self._parse_coordinates(geometry))
 
-        for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
+        return coordinates
-            if cost:
+
-                costs[idx] = cost * unit_factor
+    @qps_retry(qps=5, provider='mapbox')
+    def _calculate_isoline(self, origin, time_ranges,
+                           profile=DEFAULT_PROFILE):
+        self._validate_time_ranges(time_ranges)
+
+        origin = '{lon},{lat}'.format(lat=origin.latitude,
+                                      lon=origin.longitude)
+
+        time_ranges.sort()
+        time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
+
+        uri = self._uri(origin, time_ranges_seconds, profile)
+
+        try:
+            response = requests.get(uri)
+
+            if response.status_code == requests.codes.ok:
+                isolines = []
+                coordinates = self._parse_isochrone_service(response.text)
+                for t, c in zip(time_ranges, coordinates):
+                    isolines.append(MapboxIsochronesResponse(c, t))
+
+                return isolines
+            elif response.status_code == requests.codes.bad_request:
+                return []
+            elif response.status_code == requests.codes.unprocessable_entity:
+                return []
             else:
-                costs[idx] = isorange
+                raise ServiceException(response.status_code, response)
-
+        except requests.Timeout as te:
-        for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
+            # In case of timeout we want to stop the job because the server
-            destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
+            # could be down
-                                           destination[ENTRY_LOCATION][1])
+            self._logger.error('Timeout connecting to Mapbox isochrone service',
-
+                               te)
-        return costs, destinations
+            raise ServiceException('Error getting isochrone data from Mapbox',
+                                   None)
+        except requests.ConnectionError as ce:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to Mapbox isochrone service',
+                               exception=ce)
+            return []
 
     def calculate_isochrone(self, origin, time_ranges,
                             profile=DEFAULT_PROFILE):
-        validate_profile(profile)
+        self._validate_profile(profile)
 
-        max_speed = MAX_SPEEDS[profile]
+        return self._calculate_isoline(origin=origin,
-
+                                       time_ranges=time_ranges,
-        isochrones = []
+                                       profile=profile)
-        for time_range in time_ranges:
-            upper_rmax = max_speed * time_range  # an upper bound for the radius
-
-            coordinates = self.calculate_isoline(origin=origin,
-                                                 isorange=time_range,
-                                                 upper_rmax=upper_rmax,
-                                                 cost_method=self._calculate_matrix_cost,
-                                                 profile=profile,
-                                                 unit_factor=UNIT_FACTOR_ISOCHRONE,
-                                                 number_of_angles=MATRIX_NUM_ANGLES,
-                                                 max_iterations=MATRIX_MAX_ITERS,
-                                                 tolerance=MATRIX_TOLERANCE)
-            isochrones.append(MapboxIsochronesResponse(coordinates,
-                                                       time_range))
-        return isochrones
 
     def calculate_isodistance(self, origin, distance_range,
                               profile=DEFAULT_PROFILE):
-        validate_profile(profile)
+        self._validate_profile(profile)
 
         max_speed = MAX_SPEEDS[profile]
         time_range = distance_range / max_speed
 
-        return self.calculate_isochrone(origin=origin,
+        return self._calculate_isoline(origin=origin,
-                                        time_ranges=[time_range],
+                                       time_ranges=[time_range],
-                                        profile=profile)[0].coordinates
+                                       profile=profile)[0].coordinates
-
-    def calculate_isoline(self, origin, isorange, upper_rmax,
-                          cost_method=_calculate_matrix_cost,
-                          profile=DEFAULT_PROFILE,
-                          unit_factor=DEFAULT_UNIT_FACTOR,
-                          number_of_angles=DEFAULT_NUM_ANGLES,
-                          max_iterations=DEFAULT_MAX_ITERS,
-                          tolerance=DEFAULT_TOLERANCE):
-        # Formally, a solution is an array of {angle, radius, lat, lon, cost}
-        # with cardinality number_of_angles
-        # we're looking for a solution in which
-        # abs(cost - isorange) / isorange <= TOLERANCE
-
-        # Initial setup
-        angles = get_angles(number_of_angles)
-        rmax = [upper_rmax] * number_of_angles
-        rmin = [0.0] * number_of_angles
-        location_estimates = [calculate_dest_location(origin, a,
-                                                      upper_rmax / 2.0)
-                              for a in angles]
-
-        # Iterate to refine the first solution
-        for i in xrange(0, max_iterations):
-            # Calculate the "actual" cost for each location estimate.
-            # NOTE: sometimes it cannot calculate the cost and returns None.
-            #   Just assume isorange and stop the calculations there
-
-            costs, destinations = cost_method(origin=origin,
-                                              targets=location_estimates,
-                                              isorange=isorange,
-                                              profile=profile,
-                                              unit_factor=unit_factor,
-                                              number_of_angles=number_of_angles)
-
-            if not costs:
-                continue
-
-            errors = [(cost - isorange) / float(isorange) for cost in costs]
-            max_abs_error = max([abs(e) for e in errors])
-            if max_abs_error <= tolerance:
-                # good enough, stop there
-                break
-
-            # let's refine the solution, binary search
-            for j in xrange(0, number_of_angles):
-
-                if abs(errors[j]) > tolerance:
-                    if errors[j] > 0:
-                        rmax[j] = (rmax[j] + rmin[j]) / 2.0
-                    else:
-                        rmin[j] = (rmax[j] + rmin[j]) / 2.0
-
-                    location_estimates[j] = calculate_dest_location(origin,
-                                                                    angles[j],
-                                                                    (rmax[j] + rmin[j]) / 2.0)
-
-        # delete points that got None
-        location_estimates_filtered = []
-        for i, c in enumerate(costs):
-            if c != isorange and c < isorange * (1 + tolerance):
-                location_estimates_filtered.append(destinations[i])
-
-        return location_estimates_filtered
 
 
 class MapboxIsochronesResponse:

server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py

@@ -1,92 +0,0 @@
-'''
-Python client for the Mapbox Time Matrix service.
-'''
-
-import requests
-from cartodb_services.metrics import Traceable
-from cartodb_services.tools.coordinates import (validate_coordinates,
-                                                marshall_coordinates)
-from cartodb_services.tools.exceptions import ServiceException
-from cartodb_services.tools.qps import qps_retry
-
-BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
-           '{coordinates}'
-           '?access_token={token}'
-           '&sources=0'  # Set the first coordinate as source...
-           '&destinations=all')  # ...and the rest as destinations
-
-NUM_COORDINATES_MIN = 2  # https://www.mapbox.com/api-documentation/#matrix
-NUM_COORDINATES_MAX = 25  # https://www.mapbox.com/api-documentation/#matrix
-
-PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
-PROFILE_DRIVING = 'driving'
-PROFILE_CYCLING = 'cycling'
-PROFILE_WALKING = 'walking'
-DEFAULT_PROFILE = PROFILE_DRIVING
-
-VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
-                  PROFILE_DRIVING,
-                  PROFILE_CYCLING,
-                  PROFILE_WALKING]
-
-ENTRY_DURATIONS = 'durations'
-ENTRY_DESTINATIONS = 'destinations'
-ENTRY_LOCATION = 'location'
-
-
-def validate_profile(profile):
-    if profile not in VALID_PROFILES:
-        raise ValueError('{profile} is not a valid profile. '
-                         'Valid profiles are: {valid_profiles}'.format(
-                             profile=profile,
-                             valid_profiles=', '.join(
-                                 [x for x in VALID_PROFILES])))
-
-
-class MapboxMatrixClient(Traceable):
-    '''
-    Python wrapper for the Mapbox Time Matrix service.
-    '''
-
-    def __init__(self, token, logger, service_params=None):
-        service_params = service_params or {}
-        self._token = token
-        self._logger = logger
-
-    def _uri(self, coordinates, profile=DEFAULT_PROFILE):
-        return BASEURI.format(profile=profile, coordinates=coordinates,
-                              token=self._token)
-
-    @qps_retry(qps=1)
-    def matrix(self, coordinates, profile=DEFAULT_PROFILE):
-        validate_profile(profile)
-        validate_coordinates(coordinates,
-                             NUM_COORDINATES_MIN, NUM_COORDINATES_MAX)
-
-        coords = marshall_coordinates(coordinates)
-
-        uri = self._uri(coords, profile)
-
-        try:
-            response = requests.get(uri)
-
-            if response.status_code == requests.codes.ok:
-                return response.text
-            elif response.status_code == requests.codes.bad_request:
-                return '{}'
-            elif response.status_code == requests.codes.unprocessable_entity:
-                return '{}'
-            else:
-                raise ServiceException(response.status_code, response)
-        except requests.Timeout as te:
-            # In case of timeout we want to stop the job because the server
-            # could be down
-            self._logger.error('Timeout connecting to Mapbox matrix service',
-                               te)
-            raise ServiceException('Error getting matrix data from Mapbox',
-                                   None)
-        except requests.ConnectionError as ce:
-            # Don't raise the exception to continue with the geocoding job
-            self._logger.error('Error connecting to Mapbox matrix service',
-                               exception=ce)
-            return '{}'

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -1,5 +1,5 @@
-from routing import MapzenRouting, MapzenRoutingResponse
+from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
-from isolines import MapzenIsolines
+from cartodb_services.mapzen.isolines import MapzenIsolines
-from geocoder import MapzenGeocoder
+from cartodb_services.mapzen.geocoder import MapzenGeocoder
-from matrix_client import MatrixClient
+from cartodb_services.mapzen.matrix_client import MatrixClient
-from isochrones import MapzenIsochrones
+from cartodb_services.mapzen.isochrones import MapzenIsochrones

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -135,7 +135,7 @@ class MapzenIsolines:
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c <> isorange:
+            if c != isorange:
                 location_estimates_filtered.append(location_estimates[i])
 
         return location_estimates_filtered

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
+from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
-from quota import QuotaService
+from cartodb_services.metrics.quota import QuotaService
-from user import UserMetricsService
+from cartodb_services.metrics.user import UserMetricsService
-from log import metrics, MetricsDataGatherer, Traceable
+from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -45,7 +45,7 @@ class ServiceConfig(object):
 
     def _get_effective_monthly_quota(self, quota_key, default=0):
         quota_from_redis = self._redis_config.get(quota_key, None)
-        if quota_from_redis and quota_from_redis <> '':
+        if quota_from_redis and quota_from_redis != '':
             return int(quota_from_redis)
         else:
             return default
@@ -676,6 +676,9 @@ class ServicesDBConfig:
             raise ConfigException('Mapbox configuration missing')
 
         mapbox_conf = json.loads(mapbox_conf_json)
+
+        # Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
+        # to `isolines` to ensure retrocompatibility
         self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
         self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
         self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -6,7 +6,10 @@ import uuid
 import plpy
 from datetime import datetime
 from contextlib import contextmanager
-from urlparse import urlparse
+try:
+    from urlparse import urlparse
+except:
+    from urllib.parse import urlparse
 
 
 @contextmanager

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -1,5 +1,5 @@
-from user import UserMetricsService
+from cartodb_services.metrics.user import UserMetricsService
-from log import MetricsDataGatherer
+from cartodb_services.metrics.log import MetricsDataGatherer
 from datetime import date
 import re
 

server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py

@@ -92,6 +92,9 @@ class MapboxIsolinesConfigBuilder(object):
 
     def get(self):
         mapbox_server_conf = self._server_conf.get('mapbox_conf')
+
+        # Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
+        # to `isolines` to ensure retrocompatibility
         mapbox_api_keys = mapbox_server_conf['matrix']['api_keys']
         mapbox_service_params = mapbox_server_conf['matrix'].get('service', {})
 

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -1,5 +1,5 @@
 from ..core.interfaces import ConfigBackendInterface
-from null_config import NullConfigStorage
+from cartodb_services.refactor.storage.null_config import NullConfigStorage
 
 
 class RedisConfigStorage(ConfigBackendInterface):

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py

@@ -18,5 +18,5 @@ class RedisConnectionBuilder():
         else:
             conn = StrictRedis(host=self._config.host, port=self._config.port,
                                db=self._config.db, retry_on_timeout=True,
-                               socket_timeout=self._config.timeout)
+                               socket_timeout=self._config.timeout, decode_responses=True)
             return conn

server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py

@@ -1,4 +1,4 @@
-from geocoder import TomTomGeocoder
+from cartodb_services.tomtom.geocoder import TomTomGeocoder
-from bulk_geocoder import TomTomBulkGeocoder
+from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
-from routing import TomTomRouting, TomTomRoutingResponse
+from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
-from isolines import TomTomIsolines, TomTomIsochronesResponse
+from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py

@@ -3,6 +3,7 @@ from requests.adapters import HTTPAdapter
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.geocoder import geocoder_error_response
 from cartodb_services.tomtom import TomTomGeocoder
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools.exceptions import ServiceException
 
 
@@ -59,6 +60,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
             self._logger.error(msg, e)
             return [geocoder_error_response(msg)] * len(searches)
 
+    @qps_retry(qps=5, provider='tomtom')
     def _send_batch(self, searches):
         body = {'batchItems': [{'query': self._query(s)} for s in searches]}
         request_params = {
@@ -76,6 +78,7 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
             self._logger.error(msg)
             raise ServiceException(msg, response)
 
+    @qps_retry(qps=5, provider='tomtom')
     def _download_results(self, location):
         stalled_retries = 0
         while True:

server/lib/python/cartodb_services/cartodb_services/tools/__init__.py

@@ -1,9 +1,9 @@
-from redis_tools import RedisConnection, RedisDBConfig
+from .redis_tools import RedisConnection, RedisDBConfig
-from coordinates import Coordinate
+from .coordinates import Coordinate
-from polyline import PolyLine
+from .polyline import PolyLine
-from log import Logger, LoggerConfig
+from .log import Logger, LoggerConfig
-from rate_limiter import RateLimiter
+from .rate_limiter import RateLimiter
-from service_manager import ServiceManager
+from .service_manager import ServiceManager
-from legacy_service_manager import LegacyServiceManager
+from .legacy_service_manager import LegacyServiceManager
-from exceptions import QuotaExceededException, RateLimitExceeded
+from .exceptions import QuotaExceededException, RateLimitExceeded
-from country import country_to_iso3
+from .country import country_to_iso3

server/lib/python/cartodb_services/cartodb_services/tools/qps.py

@@ -1,7 +1,7 @@
 import time
 import random
 from datetime import datetime
-from exceptions import TimeoutException
+from cartodb_services.tools.exceptions import TimeoutException
 import re
 
 DEFAULT_RETRY_TIMEOUT = 60

server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py

@@ -23,7 +23,7 @@ class RedisConnection:
         else:
             conn = StrictRedis(host=self._config.host, port=self._config.port,
                                db=self._config.db, retry_on_timeout=True,
-                               socket_timeout=self._config.timeout)
+                               socket_timeout=self._config.timeout, decode_responses=True)
             return conn
 
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.22.0',
+    version='0.23.1',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_mapboxisoline.py

@@ -1,20 +1,27 @@
 import unittest
 from mock import Mock
-from cartodb_services.mapbox.isolines import MapboxIsolines
+from cartodb_services.mapbox.isolines import MapboxIsolines, DEFAULT_PROFILE
-from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
-from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
 from cartodb_services.tools import Coordinate
+
 from credentials import mapbox_api_key
 
 VALID_ORIGIN = Coordinate(-73.989, 40.733)
 
 
-@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
 class MapboxIsolinesTestCase(unittest.TestCase):
 
     def setUp(self):
-        matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock())
+        self.mapbox_isolines = MapboxIsolines(apikey=mapbox_api_key(),
-        self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock())
+                                              logger=Mock())
+
+    def test_invalid_time_range(self):
+        time_ranges = [4000]
+
+        with self.assertRaises(ValueError):
+            solution = self.mapbox_isolines.calculate_isochrone(
+                origin=VALID_ORIGIN,
+                profile=DEFAULT_PROFILE,
+                time_ranges=time_ranges)
 
     def test_calculate_isochrone(self):
         time_ranges = [300, 900]

server/lib/python/cartodb_services/test/test_mapboxmatrix.py

@@ -1,58 +0,0 @@
-import unittest
-from mock import Mock
-from cartodb_services.mapbox import MapboxMatrixClient
-from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
-from cartodb_services.tools.exceptions import ServiceException
-from cartodb_services.tools import Coordinate
-from credentials import mapbox_api_key
-
-INVALID_TOKEN = 'invalid_token'
-VALID_ORIGIN = Coordinate(-73.989, 40.733)
-VALID_TARGET = Coordinate(-74, 40.733)
-VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET]
-NUM_COORDINATES_MAX = 25
-INVALID_COORDINATES_EMPTY = []
-INVALID_COORDINATES_MIN = [VALID_ORIGIN]
-INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \
-                          [VALID_TARGET
-                           for x in range(0, NUM_COORDINATES_MAX + 1)]
-VALID_PROFILE = DEFAULT_PROFILE
-INVALID_PROFILE = 'invalid_profile'
-
-
-@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
-class MapboxMatrixTestCase(unittest.TestCase):
-    def setUp(self):
-        self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),
-                                                logger=Mock())
-
-    def test_invalid_profile(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(VALID_COORDINATES,
-                                      INVALID_PROFILE)
-
-    def test_invalid_coordinates_empty(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_EMPTY,
-                                      VALID_PROFILE)
-
-    def test_invalid_coordinates_max(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_MAX,
-                                      VALID_PROFILE)
-
-    def test_invalid_coordinates_min(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_MIN,
-                                      VALID_PROFILE)
-
-    def test_invalid_token(self):
-        invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock())
-        with self.assertRaises(ServiceException):
-            invalid_matrix.matrix(VALID_COORDINATES,
-                                  VALID_PROFILE)
-
-    def test_valid_request(self):
-        distance_matrix = self.matrix_client.matrix(VALID_COORDINATES,
-                                                    VALID_PROFILE)
-        assert distance_matrix

test/helpers/import_helper.py

@@ -20,7 +20,7 @@ class ImportHelper:
         response.raise_for_status()
         response_json = json.loads(response.text)
         if not response_json['success']:
-            print "Error importing the test dataset: {0}".format(response.text)
+            print("Error importing the test dataset: {0}".format(response.text))
             sys.exit(1)
         while(True):
             table_name = ImportHelper.get_imported_table_name(
@@ -42,8 +42,8 @@ class ImportHelper:
             schema, username, host, import_id, api_key)
         import_data_response = requests.get(import_url)
         if import_data_response.status_code != 200:
-            print "Error getting the table name from " \
+            print("Error getting the table name from " \
-                "the import data: {0}".format(import_data_response.text)
+                "the import data: {0}".format(import_data_response.text))
             sys.exit(1)
         import_data_json = json.loads(import_data_response.text)
 
@@ -57,5 +57,5 @@ class ImportHelper:
         )
         response = requests.get(url)
         if response.status_code != 200:
-            print "Error cleaning the test dataset: {0}".format(response.text)
+            print("Error cleaning the test dataset: {0}".format(response.text))
             sys.exit(1)

test/helpers/integration_test_helper.py

@@ -10,7 +10,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
     return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
 
 
-def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
+def assert_close_enough(xy_a, xy_b, rel_tol=0.001, abs_tol=0.0005):
     """
     Asserts that the given points are "close enough", in a square.
     :param xy_a: Array of 2 elements, X and Y.
@@ -23,7 +23,7 @@ def assert_close_enough(xy_a, xy_b, rel_tol=0.0001, abs_tol=0.0005):
         assert_true(isclose(xy_a[i], xy_b[i], rel_tol, abs_tol),
                     "Coord {} error: {} and {} are not closer than {}, {}".format(
                         i, xy_a[i], xy_b[i], rel_tol, abs_tol
-                    ))
+        ))
 
 
 class IntegrationTestHelper:
@@ -45,11 +45,15 @@ class IntegrationTestHelper:
         }
 
     @classmethod
-    def execute_query_raw(cls, sql_api_url, query):
+    def execute_query_raw(cls, sql_api_url, query, method='GET'):
         requests.packages.urllib3.disable_warnings()
-        query_url = "{0}?q={1}".format(sql_api_url, query)
+        if method.upper() == 'GET':
-        print "Executing query: {0}".format(query_url)
+            query_url = "{0}?q={1}".format(sql_api_url, query)
-        query_response = requests.get(query_url)
+            query_response = requests.get(query_url)
+            print("Executing query GET: {0}".format(query_url))
+        else:
+            query_response = requests.post(sql_api_url, data={"q": query})
+            print("Executing query POST: {0}".format(sql_api_url))
         if query_response.status_code != 200:
             raise Exception(json.loads(query_response.text)['error'])
         return json.loads(query_response.text)

test/integration/test_street_functions.py

@@ -85,9 +85,9 @@ class TestStreetFunctionsSetUp(TestCase):
 
     TOMTOM_METADATAS = {
         'Plaza España, Barcelona':
-            {'relevance': 0.85, 'precision': 'precise', 'match_types': ['street']},
+            {'relevance': 0.72, 'precision': 'precise', 'match_types': ['street']},
         'Santiago Rusiñol 123, Valladolid':
-            {'relevance': 0.45, 'precision': 'interpolated', 'match_types': ['street']}
+            {'relevance': 0.74, 'precision': 'precise', 'match_types': ['street']}
     }
 
     MAPBOX_METADATAS = {
@@ -151,10 +151,9 @@ class TestStreetFunctions(TestStreetFunctionsSetUp):
         query = "SELECT cdb_dataservices_client.cdb_geocode_street_point(street) " \
                 "as geometry FROM {0} LIMIT 1".format(table)
         try:
-            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+            self._run_authenticated(query)['rows'][0]
         except Exception as e:
-            assert_equal(e.message[0],
+            assert_equal(e, "permission denied for relation {}".format(table))
-                         "permission denied for relation {}".format(table))
 
     def test_component_aggregation(self):
         query = "select st_x(the_geom), st_y(the_geom) from (" \
@@ -335,7 +334,7 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
                 "{}" \
                 "]''::jsonb) as (cartodb_id integer, address text)', " \
                 "'address', null, null, null, {})".format(','.join(streets), batch_size)
-        response = self._run_authenticated(query)
+        response = self._run_authenticated(query, method='POST')
         assert_equal(n, len(response['rows']))
         for row in response['rows']:
             assert_not_equal(row['st_x'], None)
@@ -413,12 +412,14 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
         for r, e in zip(response['rows'], expected):
             self.assert_metadata(r['metadata'], e)
 
-    def _run_authenticated(self, query):
+    def _run_authenticated(self, query, method='GET'):
-        authenticated_query = "{}&api_key={}".format(query,
+        api_key = self.env_variables['api_key']
-                                                     self.env_variables[
+        url = self.sql_api_url
-                                                         'api_key'])
+        auth_query = "{}&api_key={}".format(query, api_key)
-        return IntegrationTestHelper.execute_query_raw(self.sql_api_url,
+        if method.upper() != 'GET':
-                                                       authenticated_query)
+            auth_query = query
+            url = "{}?api_key={}".format(self.sql_api_url, api_key)
+        return IntegrationTestHelper.execute_query_raw(url, auth_query, method)
 
     @staticmethod
     def _x_y_by_cartodb_id(response):

test/run_tests.py

@@ -36,7 +36,7 @@ def main():
         set_environment_variables(username, api_key, table_name, host, schema)
         execute_tests()
     except Exception as e:
-        print e.message
+        print(e)
         sys.exit(1)
     finally:
         clean_environment_variables()
@@ -45,11 +45,11 @@ def main():
 
 
 def usage():
-    print """Usage: run_tests.py [options] username api_key
+    print("""Usage: run_tests.py [options] username api_key
         Options:
         -h: Show this help
         --host: take that host as base (by default is cartodb.com)
-        --schema: define the url schema [http/https] (by default https)"""
+        --schema: define the url schema [http/https] (by default https)""")
 
 
 def execute_tests():
@@ -59,7 +59,7 @@ def execute_tests():
         stderr=subprocess.PIPE
     )
     out, err = process.communicate()
-    print err
+    print(err.decode('utf-8'))
     regexp = re.compile(r'FAILED \(.*\)')
     if regexp.search(err) is not None:
         sys.exit(1)