Compare commits

..

27 Commits

Author SHA1 Message Date
Javier Torres
f996cb35db Merge pull request #541 from CartoDB/development
Python 0.21.1: Fix batch geocoder precision
2018-12-26 12:18:29 +01:00
Javier Torres
bbfa03749a Merge pull request #539 from CartoDB/s1891-bulk_geocoder_no_street_address
S1891 bulk geocoder no street address
2018-12-26 12:17:48 +01:00
Javier Torres
84276b4cd3 Do not crash when matchType is empty 2018-12-24 13:04:54 +01:00
Mario de Frutos
b608830bce Merge pull request #537 from CartoDB/development
Release 0.35.0 for server extension
2018-11-30 12:34:38 +01:00
Mario de Frutos
a8140e53c3 Update routing mode type doc 2018-11-30 12:30:40 +01:00
Mario de Frutos
c28ca4861a Update NEWS.md 2018-11-30 12:24:47 +01:00
Mario de Frutos
b269c46724 Merge pull request #536 from CartoDB/add_mode_type_tomtom
Mode type options for routing
2018-11-30 12:17:06 +01:00
Mario de Frutos
6b6fbef586 Remove unused import 2018-11-30 11:31:46 +01:00
Mario de Frutos
eb4638a326 CR fixes 2018-11-30 11:24:12 +01:00
Mario de Frutos
a919b87664 Include test for valid request with route type 2018-11-29 16:22:34 +01:00
Mario de Frutos
98a4ed81c5 Add test for route type verification 2018-11-29 16:16:22 +01:00
Mario de Frutos
0ec1e051be Include options parsing functions 2018-11-29 14:28:54 +01:00
Mario de Frutos
850497ef79 Fix some missing arguments in the functions and add drop for the old ones 2018-11-29 13:02:18 +01:00
Mario de Frutos
8d2d0ececd SQL server extension artifact for version 0.35.0 2018-11-29 12:34:56 +01:00
Mario de Frutos
90fd8587e2 Upgrade and downgrade files 2018-11-29 12:12:19 +01:00
Mario de Frutos
64a0616c97 Added options and units parameters to the routing functions 2018-11-29 12:12:05 +01:00
Mario de Frutos
b1bdf2460e Version 0.35.0 initial files 2018-11-29 12:09:48 +01:00
Mario de Frutos
04bbb32849 Added mode type to Tomtom routing functions 2018-11-29 11:51:59 +01:00
Mario de Frutos
3e6c273b48 Merge pull request #535 from CartoDB/development
Release 0.34.2
2018-11-27 18:04:11 +01:00
Mario de Frutos
522591dd63 Update NEWS.md 2018-11-27 17:51:45 +01:00
Mario de Frutos
75d0959aff Merge pull request #532 from CartoDB/fix_metrics
Fix metrics
2018-11-27 17:49:55 +01:00
Mario de Frutos
b2cdb378fb Regenerate 0.34.2 artifact 2018-11-27 17:43:02 +01:00
Mario de Frutos
100319ed0e Migrate old versions 2018-11-27 17:41:14 +01:00
Mario de Frutos
5e161b55d2 Merge branch 'development' into fix_metrics 2018-11-27 17:39:56 +01:00
Mario de Frutos
7f4b87f513 Rename to release 2018-11-27 17:39:05 +01:00
Mario de Frutos
2f6d067b4e Parse JSON parameters to avoid problems
If we pass JSON as string, with scape characters and so on, we are
going to have problems parsing that metrics in ES
2018-11-05 17:06:52 +01:00
Mario de Frutos
089d3e3f27 New version 0.34.1 2018-11-05 16:51:17 +01:00
22 changed files with 8202 additions and 32 deletions

10
NEWS.md
View File

@@ -1,3 +1,13 @@
Nov 30th, 2018
==============
* Version `0.35.0` of the server extension
* Now the `mode_type` is working the providers that support it (#536)
Nov 27th, 2018
==============
* Version `0.34.2` of the server extension
* Fixed bulk geocoding log metrics in order to fix problems in ELK due to the JSON not being standard (#530)
Nov 22th, 2018
==============
* Version `0.34.1` of the server extension

View File

@@ -81,4 +81,4 @@ The optional value parameters must be passed using the format: `option=value`. N
Name | Type | Description | Accepted values
--- | --- | --- | ---
`mode_type` | `text` | Type of route calculation | `shortest` (this option only applies to the car transport mode)
`mode_type` | `text` | Type of route calculation | `shortest` or `fastest` (this option only applies to the car transport mode)

View File

@@ -0,0 +1,221 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxRouting
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.tools.normalize import options_to_dict
from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if not waypoints or len(waypoints) < 2:
service_manager.logger.info("Empty origin or destination")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
if len(waypoints) > 25:
service_manager.logger.info("Too many waypoints (max 25)")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
options_dict = options_to_dict(options)
if 'mode_type' in options_dict:
plpy.warning('Mapbox provider doesnt support route type parameter')
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
service_manager.quota_service.increment_success_service_use()
return [shape_linestring, resp.length, int(round(resp.duration))]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate Mapbox routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.tomtom import TomTomRouting
from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.tools.normalize import options_to_dict
from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
if not waypoints or len(waypoints) < 2:
service_manager.logger.info("Empty origin or destination")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
if len(waypoints) > 25:
service_manager.logger.info("Too many waypoints (max 25)")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
route_type = DEFAULT_ROUTE_TYPE
options_dict = options_to_dict(options)
if 'mode_type' in options_dict:
route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
service_manager.quota_service.increment_success_service_use()
return [shape_linestring, resp.length, int(round(resp.duration))]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate TomTom routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
username TEXT,
orgname TEXT,
origin geometry(Point, 4326),
destination geometry(Point, 4326),
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
with metrics('cdb_route_with_point', user_routing_config, logger, params):
waypoints = [origin, destination]
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

View File

@@ -0,0 +1,209 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(text, text, geometry(Point, 4326)[], text, text[], text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT)
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxRouting
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
if not waypoints or len(waypoints) < 2:
service_manager.logger.info("Empty origin or destination")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
if len(waypoints) > 25:
service_manager.logger.info("Too many waypoints (max 25)")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
service_manager.quota_service.increment_success_service_use()
return [shape_linestring, resp.length, int(round(resp.duration))]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate Mapbox routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT)
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.tomtom import TomTomRouting
from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
if not waypoints or len(waypoints) < 2:
service_manager.logger.info("Empty origin or destination")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
if len(waypoints) > 25:
service_manager.logger.info("Too many waypoints (max 25)")
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
waypoint_coords = []
for waypoint in waypoints:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
service_manager.quota_service.increment_success_service_use()
return [shape_linestring, resp.length, int(round(resp.duration))]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
else:
service_manager.quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate TomTom routing')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
username TEXT,
orgname TEXT,
origin geometry(Point, 4326),
destination geometry(Point, 4326),
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
with metrics('cdb_route_with_point', user_routing_config, logger, params):
waypoints = [origin, destination]
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.34.1'
default_version = '0.35.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

View File

@@ -0,0 +1,39 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:
provider_function = "_cdb_bulk_google_geocode_street_point";
elif user_geocoder_config.heremaps_geocoder:
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
elif user_geocoder_config.tomtom_geocoder:
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
elif user_geocoder_config.mapbox_geocoder:
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
else:
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

View File

@@ -0,0 +1,39 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searches': searches}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:
provider_function = "_cdb_bulk_google_geocode_street_point";
elif user_geocoder_config.heremaps_geocoder:
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
elif user_geocoder_config.tomtom_geocoder:
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
elif user_geocoder_config.mapbox_geocoder:
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
else:
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

File diff suppressed because it is too large Load Diff

View File

@@ -8,13 +8,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT)
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxRouting
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.tools.normalize import options_to_dict
from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder
import cartodb_services
@@ -43,6 +46,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
options_dict = options_to_dict(options)
if 'mode_type' in options_dict:
plpy.warning('Mapbox provider doesnt support route type parameter')
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
@@ -69,13 +75,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoi
username TEXT,
orgname TEXT,
waypoints geometry(Point, 4326)[],
mode TEXT)
mode TEXT,
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.tomtom import TomTomRouting
from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM, DEFAULT_ROUTE_TYPE, MODE_TYPE_TO_TOMTOM
from cartodb_services.tools import Coordinate
from cartodb_services.tools.polyline import polyline_to_linestring
from cartodb_services.tools.normalize import options_to_dict
from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
import cartodb_services
@@ -104,8 +113,12 @@ RETURNS cdb_dataservices_server.simple_route AS $$
waypoint_coords.append(Coordinate(lon,lat))
profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
route_type = DEFAULT_ROUTE_TYPE
options_dict = options_to_dict(options)
if 'mode_type' in options_dict:
route_type = MODE_TYPE_TO_TOMTOM.get(options_dict['mode_type'])
resp = client.directions(waypoint_coords, profile)
resp = client.directions(waypoint_coords, profile=profile, route_type=route_type)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:

View File

@@ -24,16 +24,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$
waypoints = [origin, destination]
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')
@@ -63,16 +63,16 @@ RETURNS cdb_dataservices_server.simple_route AS $$
with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
if user_routing_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.mapbox_provider:
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
elif user_routing_config.tomtom_provider:
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
else:
raise Exception('Requested routing method is not available')

View File

@@ -9,6 +9,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_poin
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -20,7 +21,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searches': searches}
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:

View File

@@ -8,7 +8,7 @@ from collections import namedtuple
from requests.adapters import HTTPAdapter
from cartodb_services import StreetPointBulkGeocoder
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response
from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response, PRECISION_INTERPOLATED
from cartodb_services.metrics import Traceable
from cartodb_services.tools.exceptions import ServiceException
@@ -137,8 +137,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
reader = csv.DictReader(root_zip.open(name), delimiter='|')
for row in reader:
if row['SeqNumber'] == '1': # First per requested data
precision = self.PRECISION_BY_MATCH_TYPE[
row.get('matchType', 'pointAddress')]
precision = self.PRECISION_BY_MATCH_TYPE.get(
row.get('matchType'), PRECISION_INTERPOLATED)
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None)
results.append((row['recId'],
[row['displayLongitude'], row['displayLatitude']],

View File

@@ -143,8 +143,8 @@ class HereMapsGeocoder(Traceable):
def _extract_metadata_from_result(self, result):
# See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses
precision = self.PRECISION_BY_MATCH_TYPE[
result.get('MatchType', 'pointAddress')]
precision = self.PRECISION_BY_MATCH_TYPE.get(
result.get('MatchType'), PRECISION_INTERPOLATED)
match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None)
return geocoder_metadata(
result['Relevance'],

View File

@@ -11,13 +11,14 @@ from cartodb_services.tools.coordinates import (validate_coordinates,
marshall_coordinates)
from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry
from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT)
from types import (DEFAULT_PROFILE, DEFAULT_ROUTE_TYPE, VALID_PROFILES, VALID_ROUTE_TYPE, DEFAULT_DEPARTAT)
BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/'
'{coordinates}'
'/json'
'?key={apikey}'
'&travelMode={travelmode}'
'&routeType={route_type}'
'&departAt={departat}'
'&computeBestOrder=true')
@@ -45,10 +46,11 @@ class TomTomRouting(Traceable):
self._logger = logger
def _uri(self, coordinates, profile=DEFAULT_PROFILE,
date_time=DEFAULT_DEPARTAT):
date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE):
uri = URITemplate(BASEURI).expand(apikey=self._apikey,
coordinates=coordinates,
travelmode=profile,
route_type=route_type,
departat=date_time)
return uri
@@ -60,6 +62,14 @@ class TomTomRouting(Traceable):
valid_profiles=', '.join(
[x for x in VALID_PROFILES])))
def _validate_route_type(self, route_type):
if route_type not in VALID_ROUTE_TYPE:
raise ValueError('{route_type} is not a valid route type. '
'Valid route types are: {valid_route_types}'.format(
route_type=route_type,
valid_route_types=', '.join(
[x for x in VALID_ROUTE_TYPE])))
def _marshall_coordinates(self, coordinates):
return ':'.join(['{lat},{lon}'.format(lat=coordinate.latitude,
lon=coordinate.longitude)
@@ -91,13 +101,14 @@ class TomTomRouting(Traceable):
@qps_retry(qps=5, provider='tomtom')
def directions(self, waypoints, profile=DEFAULT_PROFILE,
date_time=DEFAULT_DEPARTAT):
date_time=DEFAULT_DEPARTAT, route_type=DEFAULT_ROUTE_TYPE):
self._validate_profile(profile)
self._validate_route_type(route_type)
validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX)
coordinates = self._marshall_coordinates(waypoints)
uri = self._uri(coordinates, profile, date_time)
uri = self._uri(coordinates, profile, date_time, route_type)
try:
response = requests.get(uri)

View File

@@ -6,12 +6,16 @@ PROFILE_DRIVING = 'car'
PROFILE_CYCLING = 'bicycle'
PROFILE_WALKING = 'pedestrian'
DEFAULT_PROFILE = PROFILE_DRIVING
ROUTE_TYPE_FAST = 'fastest'
ROUTE_TYPE_SHORT = 'shortest'
DEFAULT_DEPARTAT = 'now'
VALID_PROFILES = [PROFILE_DRIVING,
PROFILE_CYCLING,
PROFILE_WALKING]
VALID_ROUTE_TYPE = [ROUTE_TYPE_SHORT,
ROUTE_TYPE_FAST]
MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
@@ -20,7 +24,13 @@ MAX_SPEEDS = {
}
TRANSPORT_MODE_TO_TOMTOM = {
'car': 'car',
'walk': 'pedestrian',
'bicycle': 'bicycle',
'car': PROFILE_DRIVING,
'walk': PROFILE_WALKING,
'bicycle': PROFILE_CYCLING,
}
DEFAULT_ROUTE_TYPE = ROUTE_TYPE_SHORT
MODE_TYPE_TO_TOMTOM = {
'shortest': ROUTE_TYPE_SHORT,
'fastest': ROUTE_TYPE_FAST
}

View File

@@ -1,3 +1,6 @@
def normalize(str_input):
return str_input.replace('&quot;', '"') \
.replace(';', ',')
def options_to_dict(options):
return dict(option.split("=") for option in options)

View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.20.2',
version='0.21.1',
description='CartoDB Services API Python Library',

View File

@@ -15,7 +15,9 @@ INVALID_WAYPOINTS_MIN = [Coordinate(13.42936, 52.50931)]
INVALID_WAYPOINTS_MAX = [Coordinate(13.42936, 52.50931)
for x in range(0, NUM_WAYPOINTS_MAX + 2)]
VALID_PROFILE = DEFAULT_PROFILE
VALID_ROUTE_TYPE = 'fastest'
INVALID_PROFILE = 'invalid_profile'
INVALID_ROUTE_TYPE = 'invalid_route_type'
class TomTomRoutingTestCase(unittest.TestCase):
@@ -43,6 +45,9 @@ class TomTomRoutingTestCase(unittest.TestCase):
with self.assertRaises(ServiceException):
invalid_routing.directions(VALID_WAYPOINTS,
VALID_PROFILE)
def test_invalid_route_type(self):
with self.assertRaises(ValueError):
self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=INVALID_ROUTE_TYPE)
def test_valid_request(self):
route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE)
@@ -50,3 +55,9 @@ class TomTomRoutingTestCase(unittest.TestCase):
assert route.shape
assert route.length
assert route.duration
route = self.routing.directions(VALID_WAYPOINTS, VALID_PROFILE, route_type=VALID_ROUTE_TYPE)
assert route.shape
assert route.length
assert route.duration