Release server 0.39.2 and python 0.23.4

Preparing release (server 0.39.2 python 0.23.4)

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v1
 
       - name: Set CLOUDSDK_PYTHON path
-        run: echo '::set-env name=CLOUDSDK_PYTHON::/usr/bin/python'
+        run: echo "CLOUDSDK_PYTHON=/usr/bin/python" >> $GITHUB_ENV
 
       - name: Setup gcloud authentication
         uses: GoogleCloudPlatform/github-actions/setup-gcloud@master

NEWS.md

@@ -1,3 +1,16 @@
+Nov 19th, 2020
+==============
+* Version `0.39.2` of the server extension
+* Version `0.23.4` of the Python library
+    * Better messages on remote errors.
+    * Fix error on unknown status codes.
+    * Here bulk geocoder: Return also errors and no-matches.
+
+Sep 23th, 2020
+==============
+* Version `0.39.1` of the server extension
+    * Allow isolines services for google-geocoding users.
+
 Sep 23th, 2020
 ==============
 * Version `0.23.3` of the Python library

docs/developer-center/reference/07-demographic-functions.md

@@ -1,5 +1,9 @@
 ### Demographic Functions
 
+<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
+  <p>We will no longer support these demographic functions from the 1st of January 2021. If you want to collect demographic reports around a point location, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
+</div>
+
 The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions]({{site.dataservicesapi_docs}}/reference/#geocoding-functions) documentation.
 
 _**Note:** The Demographic Snapshot functions are only available for the United States._

docs/developer-center/reference/09-segmentation-functions.md

@@ -1,5 +1,9 @@
 ## Segmentation Functions
 
+<div class="is-caption is-caption--mobile header-warning is-semibold u-vspace--16">
+  <p>We will no longer support these segmentations functions from the 1st of January 2021. For a similar functionality, we recommend using Data Observatory from our Python library <a href="https://carto.com/developers/cartoframes/">CARTOframes</a> instead.</p>
+</div>
+
 The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions]({{site.dataservicesapi_docs}}/reference/#geocoding-functions) documentation.
 
 _**Note:** The Segmentation Snapshot functions are only available for the United States. Our first release (May 18, 2016) is derived from Census 2010 variables. Our next release will be based on Census 2014 data. For the latest information, see the [Open Segments](https://github.com/CartoDB/open-segments) project repository._

server/extension/Makefile

@@ -1,7 +1,7 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = 0.39.0
+EXTVERSION = 0.39.2
 
 # The new version to be generated from templates
 SED ?= sed

server/extension/cdb_dataservices_server--0.39.1--0.39.2.sql

@@ -0,0 +1,140 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing: ' + str(e))
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
+
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing: ' + str(e))
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.39.2--0.39.1.sql

@@ -0,0 +1,139 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
+
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.39.0--0.39.1.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.39.1--0.39.0.sql

@@ -0,0 +1,76 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/sql/100_routing_helper.sql

@@ -66,11 +66,12 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     import sys
     service_manager.quota_service.increment_failed_service_use()
     service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to calculate Mapbox routing')
+    raise Exception('Error trying to calculate Mapbox routing: ' + str(e))
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
   username TEXT,
   orgname TEXT,
@@ -134,7 +135,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     import sys
     service_manager.quota_service.increment_failed_service_use()
     service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to calculate TomTom routing')
+    raise Exception('Error trying to calculate TomTom routing: ' + str(e))
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE @@plpythonu@@ SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/85_isodistance.sql

@@ -11,9 +11,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  if user_isolines_config.google_services_user:
-    raise Exception('This service is not available for google service users.')
-
   params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
 
   with metrics('cdb_isodistance', user_isolines_config, logger, params):

server/extension/sql/90_isochrone.sql

@@ -11,9 +11,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  if user_isolines_config.google_services_user:
-    raise Exception('This service is not available for google service users.')
-
   params = {'username': username, 'orgname': orgname, 'source': source, 'mode': mode, 'range': range, 'options': options}
 
   with metrics('cdb_isochrone', user_isolines_config, logger, params):

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -126,7 +126,7 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
             status=polling_root.find('./Response/Status').text)
 
     def _download_results(self, job_id):
-        result_r = self.session.get("{}/{}/result".format(self.BATCH_URL, job_id),
+        result_r = self.session.get("{}/{}/all".format(self.BATCH_URL, job_id),
                                     params=self.credentials_params,
                                     timeout=(self.connect_timeout, self.read_timeout))
         root_zip = zipfile.ZipFile(io.BytesIO(result_r.content))
@@ -147,6 +147,9 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
                                             precision,
                                             [match_type] if match_type else []
                                         )))
-
+                    elif row['matchLevel'] == 'NOMATCH':
+                        results.append((row['recId'], [], {}))
+                    elif row['matchLevel'] == 'FAILED':
+                        results.append((row['recId'], [], {'error': 'Bulk geocoder failed'}))
         return results
 

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -105,7 +105,7 @@ class MapboxIsolines():
             elif response.status_code == requests.codes.unprocessable_entity:
                 return []
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/mapbox/routing.py

@@ -94,7 +94,7 @@ class MapboxRouting(Traceable):
             elif response.status_code == requests.codes.unprocessable_entity:
                 return MapboxRoutingResponse(None, None, None)
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py

@@ -82,7 +82,7 @@ class TomTomIsolines():
             elif response.status_code == requests.codes.unprocessable_entity:
                 return []
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py

@@ -120,7 +120,7 @@ class TomTomRouting(Traceable):
             elif response.status_code == requests.codes.unprocessable_entity:
                 return TomTomRoutingResponse(None, None, None)
             else:
-                raise ServiceException(response.status_code, response)
+                raise ServiceException('Unexpected response (' + str(response.status_code) + '): ' + str(response.content), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down

server/lib/python/cartodb_services/requirements.txt

@@ -6,6 +6,9 @@ rollbar==0.13.2
 # Dependency for googlemaps package
 requests==2.20.0
 rratelimit==0.0.4
+# botocore 1.19.0 introduced an error `ImportError: cannot import name 'IPV6_ADDRZ_RE'`
+# and mapbox depends on botocore<=1.19.0 so forcing 1.18.18
+botocore==1.18.18
 mapbox==0.14.0
 pygeocodio==0.11.1
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.23.3',
+    version='0.23.4',
 
     description='CartoDB Services API Python Library',