Merge pull request #541 from CartoDB/development

Python 0.21.1: Fix batch geocoder precision

NEWS.md

@@ -1,3 +1,29 @@
+Nov 30th, 2018
+==============
+* Version `0.35.0` of the server extension
+    * Now the `mode_type` is working the providers that support it (#536)
+
+Nov 27th, 2018
+==============
+* Version `0.34.2` of the server extension
+    * Fixed bulk geocoding log metrics in order to fix problems in ELK due to the JSON not being standard (#530)
+
+Nov 22th, 2018
+==============
+* Version `0.34.1` of the server extension
+    * Fixed isochrones metrics that increase `success_rows` field for every isochrone generated (#533)
+
+Oct 31th, 2018
+==============
+* Version `0.20.2` of the python library
+    * Added missing provider property to the QPS decorator in other TomTom services
+    * Now we only retry with the properly header coming from TomTom
+
+Oct 3rd, 2018
+==============
+* Version `0.20.1` of the python library
+    * Fix QPS manager to retry with 403 status codes coming from TomTom
+
 Sep 13th, 2018
 ==============
 * Version `0.34.0` of the server, and `0.26.0` of the client.

doc/isoline_functions.md

@@ -16,6 +16,8 @@ The following functions provide an isoline generator service, based on time or d
 
 Displays a contoured line on a map, connecting geometries to a defined area, measured by an equal range of distance (in meters).
 
+Note that not all the providers, for example TomTom, provide us a way to define the isoline limit in distance so we need to make some estimations. Due that estimations the produced isolines could not be 100% precise.
+
 #### Arguments
 
 Name | Type | Description | Accepted values

doc/routing_functions.md

@@ -81,4 +81,4 @@ The optional value parameters must be passed using the format: `option=value`. N
 
 Name | Type | Description | Accepted values
 --- | --- | --- | ---
-`mode_type` | `text` | Type of route calculation | `shortest` (this option only applies to the car transport mode)
+`mode_type` | `text` | Type of route calculation | `shortest` or `fastest` (this option only applies to the car transport mode)

server/extension/cdb_dataservices_server--0.34.2--0.35.0.sql

@@ -0,0 +1,221 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
+
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
+  username TEXT,
+  orgname TEXT,
+  origin geometry(Point, 4326),
+  destination geometry(Point, 4326),
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_point', user_routing_config, logger, params):
+    waypoints = [origin, destination]
+
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.35.0--0.34.2.sql

@@ -0,0 +1,209 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT)
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxRouting
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate Mapbox routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT)
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
+  username TEXT,
+  orgname TEXT,
+  origin geometry(Point, 4326),
+  destination geometry(Point, 4326),
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_point', user_routing_config, logger, params):
+    waypoints = [origin, destination]
+
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.34.0'
+default_version = '0.35.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.34.0--0.34.1.sql

@@ -0,0 +1,182 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  from cartodb_services.mapzen.types import coordinates_to_polygon
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  # -- Check the quota
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
+                                         logger, user_isolines_routing_config.mapzen_isochrones_service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    resp = mapzen_isochrones.isochrone(origin, mode, data_range)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      quota_service.increment_success_service_use()
+      quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isochrones')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.0.sql

@@ -0,0 +1,188 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  from cartodb_services.mapzen.types import coordinates_to_polygon
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  # -- Check the quota
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
+                                         logger, user_isolines_routing_config.mapzen_isochrones_service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    resp = mapzen_isochrones.isochrone(origin, mode, data_range)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      quota_service.increment_success_service_use()
+      quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isochrones')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          service_manager.quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          service_manager.quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          service_manager.quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          service_manager.quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.34.1--0.34.2.sql

@@ -0,0 +1,39 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.34.2--0.34.1.sql

@@ -0,0 +1,39 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': searches}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/100_routing_helper.sql

@@ -8,13 +8,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi
   username TEXT,
   orgname TEXT,
   waypoints geometry(Point, 4326)[],
-  mode TEXT)
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
 RETURNS cdb_dataservices_server.simple_route AS $$
   from cartodb_services.tools import ServiceManager
   from cartodb_services.mapbox import MapboxRouting
   from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
   from cartodb_services.tools import Coordinate
   from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
   from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
 
   import cartodb_services
@@ -43,6 +46,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       waypoint_coords.append(Coordinate(lon,lat))
 
     profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      plpy.warning('Mapbox provider doesnt support route type parameter')
 
     resp = client.directions(waypoint_coords, profile)
     if resp and resp.shape:
@@ -69,13 +75,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoi
   username TEXT,
   orgname TEXT,
   waypoints geometry(Point, 4326)[],
-  mode TEXT)
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
 RETURNS cdb_dataservices_server.simple_route AS $$
   from cartodb_services.tools import ServiceManager
   from cartodb_services.tomtom import TomTomRouting
-  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
   from cartodb_services.tools import Coordinate
   from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.tools.normalize import options_to_dict
   from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
 
   import cartodb_services
@@ -104,8 +113,12 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       waypoint_coords.append(Coordinate(lon,lat))
 
     profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+    route_type = DEFAULT_ROUTE_TYPE
+    options_dict = options_to_dict(options)
+    if 'mode_type' in options_dict:
+      route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
 
-    resp = client.directions(waypoint_coords, profile)
+    resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
     if resp and resp.shape:
       shape_linestring = polyline_to_linestring(resp.shape)
       if shape_linestring:

server/extension/sql/105_route_between_points.sql

@@ -24,16 +24,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     waypoints = [origin, destination]
 
     if user_routing_config.mapzen_provider:
-      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     elif user_routing_config.mapbox_provider:
-      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     elif user_routing_config.tomtom_provider:
-      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
@@ -63,16 +63,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$
 
   with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
     if user_routing_config.mapzen_provider:
-      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     elif user_routing_config.mapbox_provider:
-      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     elif user_routing_config.tomtom_provider:
-      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
-      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')

server/extension/sql/21_bulk_geocode_street.sql

@@ -9,6 +9,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_poin
 RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   from cartodb_services.metrics import metrics
   from cartodb_services.tools import Logger
+  import json
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -20,7 +21,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  params = {'username': username, 'orgname': orgname, 'searches': searches}
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
 
   with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
     if user_geocoder_config.google_geocoder:

server/extension/sql/80_isolines_helper.sql

@@ -295,10 +295,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       for isochrone in resp:
         result_polygon = coordinates_to_polygon(isochrone.coordinates)
         if result_polygon:
-          quota_service.increment_success_service_use()
           result.append([source, isochrone.duration, result_polygon])
         else:
-          quota_service.increment_empty_service_use()
           result.append([source, isochrone.duration, None])
       quota_service.increment_success_service_use()
       quota_service.increment_isolines_service_use(len(result))
@@ -356,10 +354,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       for isochrone in resp:
         result_polygon = coordinates_to_polygon(isochrone.coordinates)
         if result_polygon:
-          service_manager.quota_service.increment_success_service_use()
           result.append([source, isochrone.duration, result_polygon])
         else:
-          service_manager.quota_service.increment_empty_service_use()
           result.append([source, isochrone.duration, None])
       service_manager.quota_service.increment_success_service_use()
       service_manager.quota_service.increment_isolines_service_use(len(result))
@@ -416,10 +412,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
       for isochrone in resp:
         result_polygon = coordinates_to_polygon(isochrone.coordinates)
         if result_polygon:
-          service_manager.quota_service.increment_success_service_use()
           result.append([source, isochrone.duration, result_polygon])
         else:
-          service_manager.quota_service.increment_empty_service_use()
           result.append([source, isochrone.duration, None])
       service_manager.quota_service.increment_success_service_use()
       service_manager.quota_service.increment_isolines_service_use(len(result))

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -8,7 +8,7 @@ from collections import namedtuple
 from requests.adapters import HTTPAdapter
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.here import HereMapsGeocoder
-from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response
+from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED
 from cartodb_services.metrics import Traceable
 from cartodb_services.tools.exceptions import ServiceException
 
@@ -137,8 +137,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
                 reader = csv.DictReader(root_zip.open(name), delimiter='|')
                 for row in reader:
                     if row['SeqNumber'] == '1':  # First per requested data
-                        precision = self.PRECISION_BY_MATCH_TYPE[
-                            row.get('matchType', 'pointAddress')]
+                        precision = self.PRECISION_BY_MATCH_TYPE.get(
+                            row.get('matchType'), PRECISION_INTERPOLATED)
                         match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None)
                         results.append((row['recId'],
                                         [row['displayLongitude'], row['displayLatitude']],

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -143,8 +143,8 @@ class HereMapsGeocoder(Traceable):
 
     def _extract_metadata_from_result(self, result):
         # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses
-        precision = self.PRECISION_BY_MATCH_TYPE[
-            result.get('MatchType', 'pointAddress')]
+        precision = self.PRECISION_BY_MATCH_TYPE.get(
+            result.get('MatchType'), PRECISION_INTERPOLATED)
         match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None)
         return geocoder_metadata(
             result['Relevance'],

server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py

@@ -70,7 +70,7 @@ class TomTomGeocoder(Traceable):
 
         return False
 
-    @qps_retry(qps=5)
+    @qps_retry(qps=5, provider='tomtom')
     def geocode(self, searchtext, city=None, state_province=None,
                 country=None):
         response = self.geocode_meta(searchtext, city, state_province, country)
@@ -80,7 +80,7 @@ class TomTomGeocoder(Traceable):
         else:
             return response[0]
 
-    @qps_retry(qps=5)
+    @qps_retry(qps=5, provider='tomtom')
     def geocode_meta(self, searchtext, city=None, state_province=None,
                 country=None):
         if searchtext:

server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py

@@ -11,13 +11,14 @@ from cartodb_services.tools.coordinates import (validate_coordinates,
                                                 marshall_coordinates)
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools.qps import qps_retry
-from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT)
+from types import (DEFAULT_PROFILE, DEFAULT_ROUTE_TYPE, VALID_PROFILES, VALID_ROUTE_TYPE, DEFAULT_DEPARTAT)
 
 BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/'
            '{coordinates}'
            '/json'
            '?key={apikey}'
            '&travelMode={travelmode}'
+           '&routeType={route_type}'
            '&departAt={departat}'
            '&computeBestOrder=true')
 
@@ -45,10 +46,11 @@ class TomTomRouting(Traceable):
         self._logger = logger
 
     def _uri(self, coordinates, profile=DEFAULT_PROFILE,
-             date_time=DEFAULT_DEPARTAT):
+             date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE):
         uri = URITemplate(BASEURI).expand(apikey=self._apikey,
                                           coordinates=coordinates,
                                           travelmode=profile,
+                                          route_type=route_type,
                                           departat=date_time)
         return uri
 
@@ -60,6 +62,14 @@ class TomTomRouting(Traceable):
                                  valid_profiles=', '.join(
                                      [x for x in VALID_PROFILES])))
 
+    def _validate_route_type(self, route_type):
+        if route_type not in VALID_ROUTE_TYPE:
+            raise ValueError('{route_type} is not a valid route type. '
+                             'Valid route types are: {valid_route_types}'.format(
+                                 route_type=route_type,
+                                 valid_route_types=', '.join(
+                                     [x for x in VALID_ROUTE_TYPE])))
+
     def _marshall_coordinates(self, coordinates):
         return ':'.join(['{lat},{lon}'.format(lat=coordinate.latitude,
                                               lon=coordinate.longitude)
@@ -89,15 +99,16 @@ class TomTomRouting(Traceable):
                                  point[ENTRY_LONGITUDE]))
         return geometry
 
-    @qps_retry(qps=5)
+    @qps_retry(qps=5, provider='tomtom')
     def directions(self, waypoints, profile=DEFAULT_PROFILE,
-                   date_time=DEFAULT_DEPARTAT):
+                   date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE):
         self._validate_profile(profile)
+        self._validate_route_type(route_type)
         validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX)
 
         coordinates = self._marshall_coordinates(waypoints)
 
-        uri = self._uri(coordinates, profile, date_time)
+        uri = self._uri(coordinates, profile, date_time, route_type)
 
         try:
             response = requests.get(uri)

server/lib/python/cartodb_services/cartodb_services/tomtom/types.py

@@ -6,12 +6,16 @@ PROFILE_DRIVING = 'car'
 PROFILE_CYCLING = 'bicycle'
 PROFILE_WALKING = 'pedestrian'
 DEFAULT_PROFILE = PROFILE_DRIVING
+ROUTE_TYPE_FAST = 'fastest'
+ROUTE_TYPE_SHORT = 'shortest'
 
 DEFAULT_DEPARTAT = 'now'
 
 VALID_PROFILES = [PROFILE_DRIVING,
                   PROFILE_CYCLING,
                   PROFILE_WALKING]
+VALID_ROUTE_TYPE = [ROUTE_TYPE_SHORT,
+                    ROUTE_TYPE_FAST]
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
@@ -20,7 +24,13 @@ MAX_SPEEDS = {
 }
 
 TRANSPORT_MODE_TO_TOMTOM = {
-    'car': 'car',
-    'walk': 'pedestrian',
-    'bicycle': 'bicycle',
+    'car': PROFILE_DRIVING,
+    'walk': PROFILE_WALKING,
+    'bicycle': PROFILE_CYCLING,
+}
+
+DEFAULT_ROUTE_TYPE = ROUTE_TYPE_SHORT
+MODE_TYPE_TO_TOMTOM = {
+    'shortest': ROUTE_TYPE_SHORT,
+    'fastest': ROUTE_TYPE_FAST
 }

server/lib/python/cartodb_services/cartodb_services/tools/normalize.py

@@ -1,3 +1,6 @@
 def normalize(str_input):
     return str_input.replace('"', '"') \
                     .replace(';', ',')
+
+def options_to_dict(options):
+    return dict(option.split("=") for option in options)

server/lib/python/cartodb_services/cartodb_services/tools/qps.py

@@ -6,6 +6,9 @@ from exceptions import TimeoutException
 DEFAULT_RETRY_TIMEOUT = 60
 DEFAULT_QUERIES_PER_SECOND = 10
 
+TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit'
+TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header'
+
 
 def qps_retry(original_function=None, **options):
     """ Query Per Second retry decorator
@@ -46,6 +49,8 @@ class QPSService:
                 response = getattr(e, 'response', None)
                 if response is not None:
                     if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403):
+                        if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER:
+                            raise e
                         self.retry(start_time, attempt_number)
                     elif response.status_code == 429:
                         self.retry(start_time, attempt_number)

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.20.1',
+    version='0.21.1',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_tomtomrouting.py

@@ -15,7 +15,9 @@ INVALID_WAYPOINTS_MIN = [Coordinate(13.42936, 52.50931)]
 INVALID_WAYPOINTS_MAX = [Coordinate(13.42936, 52.50931)
                          for x in range(0, NUM_WAYPOINTS_MAX + 2)]
 VALID_PROFILE = DEFAULT_PROFILE
+VALID_ROUTE_TYPE = 'fastest'
 INVALID_PROFILE = 'invalid_profile'
+INVALID_ROUTE_TYPE = 'invalid_route_type'
 
 
 class TomTomRoutingTestCase(unittest.TestCase):
@@ -43,6 +45,9 @@ class TomTomRoutingTestCase(unittest.TestCase):
         with self.assertRaises(ServiceException):
             invalid_routing.directions(VALID_WAYPOINTS,
                                        VALID_PROFILE)
+    def test_invalid_route_type(self):
+        with self.assertRaises(ValueError):
+            self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=INVALID_ROUTE_TYPE)
 
     def test_valid_request(self):
         route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE)
@@ -50,3 +55,9 @@ class TomTomRoutingTestCase(unittest.TestCase):
         assert route.shape
         assert route.length
         assert route.duration
+
+        route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=VALID_ROUTE_TYPE)
+
+        assert route.shape
+        assert route.length
+        assert route.duration