From 79d0b5ba7c59e23c63cee32bf99dde17a76b64a6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 21 Jul 2016 13:46:57 +0200 Subject: [PATCH] Add service providers that come from the user configuration --- server/extension/Makefile | 2 +- ...db_dataservices_server--0.13.2--0.13.3.sql | 240 ++++++++++++++++++ ...db_dataservices_server--0.13.3--0.13.2.sql | 223 ++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- ...db_dataservices_server--0.13.1--0.13.2.sql | 0 ...db_dataservices_server--0.13.2--0.13.1.sql | 0 .../cdb_dataservices_server--0.13.2.sql | 0 server/extension/sql/15_config_helper.sql | 28 -- server/extension/sql/20_geocode_street.sql | 13 +- server/extension/sql/80_isolines_helper.sql | 6 +- server/extension/sql/85_isodistance.sql | 28 +- server/extension/sql/90_isochrone.sql | 37 ++- .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 210 +++++++-------- server/lib/python/cartodb_services/setup.py | 2 +- .../cartodb_services/test/test_helper.py | 4 +- .../test/test_quota_service.py | 27 +- 17 files changed, 645 insertions(+), 179 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.13.2--0.13.3.sql create mode 100644 server/extension/cdb_dataservices_server--0.13.3--0.13.2.sql rename server/extension/{ => old_versions}/cdb_dataservices_server--0.13.1--0.13.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.13.2--0.13.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.13.2.sql (100%) diff --git a/server/extension/Makefile b/server/extension/Makefile index e074918..dcce17f 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -8,7 +8,7 @@ ERB = erb REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql -REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) +REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) TEST_DIR = test/ REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' diff --git a/server/extension/cdb_dataservices_server--0.13.2--0.13.3.sql b/server/extension/cdb_dataservices_server--0.13.2--0.13.3.sql new file mode 100644 index 0000000..f3535ec --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.13.2--0.13.3.sql @@ -0,0 +1,240 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + if user_isolines_config.heremaps_provider: + plpy.notice('Requested isolines provider is heremaps') + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1) + elif user_isolines_config.mapzen_provider: + plpy.notice('Requested isolines provider is mapzen') + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1) + else: + plpy.error('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + + +-- isochrones + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + if user_isolines_config.heremaps_provider: + plpy.notice('Requested isolines provider is heremaps') + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1) + elif user_isolines_config.mapzen_provider: + plpy.notice('Requested isolines provider is mapzen') + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1) + else: + plpy.error('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- mapzen isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines( + username TEXT, + orgname TEXT, + isotype TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient + from cartodb_services.mapzen import MapzenIsolines + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key) + mapzen_isolines = MapzenIsolines(client) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + if isotype == 'isodistance': + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + elif isotype == 'isochrone': + for r in data_range: + isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) + plpy.debug(traceback.format_tb(traceback_)) + raise e + #plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.13.3--0.13.2.sql b/server/extension/cdb_dataservices_server--0.13.3--0.13.2.sql new file mode 100644 index 0000000..ece562c --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.13.3--0.13.2.sql @@ -0,0 +1,223 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit + +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenGeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenIsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_isolines_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + # The configuration is retrieved but no checks are performed on it + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +-- isodistance + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +-- isochrones + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + type = 'isochrone' + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) + + return result +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines( + username TEXT, + orgname TEXT, + isotype TEXT, + source geometry(Geometry, 4326), + mode TEXT, + data_range integer[], + options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.mapzen import MatrixClient + from cartodb_services.mapzen import MapzenIsolines + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key) + mapzen_isolines = MapzenIsolines(client) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + origin = {'lat': lat, 'lon': lon} + else: + raise Exception('source is NULL') + + # -- TODO Support options properly + isolines = {} + if isotype == 'isodistance': + for r in data_range: + isoline = mapzen_isolines.calculate_isodistance(origin, mode, r) + isolines[r] = isoline + elif isotype == 'isochrone': + for r in data_range: + isoline = mapzen_isolines.calculate_isochrone(origin, mode, r) + isolines[r] = isoline + + result = [] + for r in data_range: + + if len(isolines[r]) >= 3: + # -- TODO encapsulate this block into a func/method + locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point + wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations]) + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + multipolygon = plpy.execute(sql, 1)[0]['geom'] + else: + multipolygon = None + + result.append([source, r, multipolygon]) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) + return result + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) + plpy.debug(traceback.format_tb(traceback_)) + raise e + #plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index a390bba..d877823 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.13.2' +default_version = '0.13.3' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.13.1--0.13.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.1--0.13.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.1--0.13.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.1--0.13.2.sql diff --git a/server/extension/cdb_dataservices_server--0.13.2--0.13.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.2--0.13.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.2--0.13.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.2--0.13.1.sql diff --git a/server/extension/cdb_dataservices_server--0.13.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.2.sql diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 118bbec..68726ff 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -12,34 +12,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_mapzen_geocoder_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import MapzenGeocoderConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = mapzen_geocoder_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text) -RETURNS boolean AS $$ - cache_key = "user_mapzen_isolines_routing_config_{0}".format(username) - if cache_key in GD: - return False - else: - from cartodb_services.metrics import MapzenIsolinesRoutingConfig - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = mapzen_isolines_config - return True -$$ LANGUAGE plpythonu SECURITY DEFINER; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 606233e..8f3dabb 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -7,12 +7,15 @@ RETURNS Geometry AS $$ user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] if user_geocoder_config.heremaps_geocoder: + plpy.notice('Requested geocoder is heremaps') here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] elif user_geocoder_config.google_geocoder: + plpy.notice('Requested geocoder is google') google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] elif user_geocoder_config.mapzen_geocoder: + plpy.notice('Requested geocoder is mapzen') mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: @@ -56,8 +59,8 @@ RETURNS Geometry AS $$ # The configuration is retrieved but no checks are performed on it plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] @@ -137,13 +140,13 @@ RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)] - quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key) + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) country_iso3 = None if country: country_iso3 = country_to_iso3(country) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 2757369..597eebd 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -70,15 +70,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.metrics import QuotaService redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] # -- Check the quota - quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn) + quota_service = QuotaService(user_isolines_routing_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') try: - client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key) + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key) mapzen_isolines = MapzenIsolines(client) if source: diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index 15dd9f4..933aa40 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -4,11 +4,31 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - type = 'isodistance' if user_isolines_config.google_services_user: plpy.error('This service is not available for google service users.') + if user_isolines_config.heremaps_provider: + plpy.notice('Requested isolines provider is heremaps') + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1) + elif user_isolines_config.mapzen_provider: + plpy.notice('Requested isolines provider is mapzen') + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1) + else: + plpy.error('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isodistance +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + type = 'isodistance' + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) @@ -20,11 +40,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(userna RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isodistance' - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) return result diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index d4c78d4..ee230f0 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -1,4 +1,27 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + + if user_isolines_config.heremaps_provider: + plpy.notice('Requested isolines provider is heremaps') + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1) + elif user_isolines_config.mapzen_provider: + plpy.notice('Requested isolines provider is mapzen') + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1) + else: + plpy.error('Requested isolines provider is not available') +$$ LANGUAGE plpythonu; + +-- heremaps isochrone +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -6,26 +29,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' - if user_isolines_config.google_services_user: - plpy.error('This service is not available for google service users.') - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + return result $$ LANGUAGE plpythonu; - --- mapzen isochrones +-- mapzen isochrone CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options]) - return result $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 78d09e7..078bbbd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 7ac7dba..986f525 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -104,17 +104,24 @@ class ObservatoryConfig(DataObservatoryConfig): class RoutingConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' + ROUTING_PROVIDER_KEY = 'routing_provider' + MAPZEN_PROVIDER = 'mapzen' + DEFAULT_PROVIDER = 'mapzen' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, username, orgname) + self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY] + if not self._routing_provider: + self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key self._monthly_quota = self._db_config.mapzen_routing_monthly_quota self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property def service_type(self): - return 'routing_mapzen' + if self._routing_provider == self.MAPZEN_PROVIDER: + return 'routing_mapzen' @property def mapzen_api_key(self): @@ -128,93 +135,57 @@ class RoutingConfig(ServiceConfig): def period_end_date(self): return self._period_end_date -# Explicit class for the geocoder configuration for Mapzen -# which does not require users to be configured to use the service -class MapzenGeocoderConfig(ServiceConfig): - - PERIOD_END_DATE = 'period_end_date' - - def __init__(self, redis_connection, db_conn, username, orgname=None): - super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn, - username, orgname) - self._log_path = self._db_config.geocoder_log_path - try: - self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key - self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota - self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) - self._cost_per_hit = 0 - except Exception as e: - raise ConfigException("Malformed config for Mapzen geocoder: {0}".format(e)) - - @property - def service_type(self): - return 'geocoder_mapzen' - - @property - def mapzen_api_key(self): - return self._mapzen_api_key - - @property - def period_end_date(self): - return self._period_end_date - - @property - def cost_per_hit(self): - return self._cost_per_hit - - @property - def google_geocoder(self): - return None - - @property - def geocoding_quota(self): - return self._monthly_quota - - @property - def soft_geocoding_limit(self): - return False - - @property - def is_high_resolution(self): - return True - - @property - def log_path(self): - return self._log_path - class IsolinesRoutingConfig(ServiceConfig): - ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', + ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_isolines_app_id', 'heremaps_isolines_app_code', - 'geocoder_type'] + 'heremaps_isolines_app_id', 'geocoder_provider', + 'heremaps_isolines_app_code', 'isolines_provider'] QUOTA_KEY = 'here_isolines_quota' SOFT_LIMIT_KEY = 'soft_here_isolines_limit' PERIOD_END_DATE = 'period_end_date' - GEOCODER_TYPE_KEY = 'geocoder_type' - GOOGLE_GEOCODER = 'google' + ISOLINES_PROVIDER_KEY = 'isolines_provider' + GEOCODER_PROVIDER_KEY = 'geocoder_provider' + MAPZEN_PROVIDER = 'mapzen' + HEREMAPS_PROVIDER = 'heremaps' + DEFAULT_PROVIDER = 'heremaps' def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn, username, orgname) - filtered_config = {key: self._redis_config[key] for key in self.ROUTING_CONFIG_KEYS if key in self._redis_config.keys()} + filtered_config = {key: self._redis_config[key] for key in self.ISOLINES_CONFIG_KEYS if key in self._redis_config.keys()} self.__parse_config(filtered_config, self._db_config) def __parse_config(self, filtered_config, db_config): - self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower() - self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) - self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) - if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': - self._soft_isolines_limit = True - else: + self._isolines_provider = filtered_config[self.ISOLINES_PROVIDER_KEY].lower() + if not self._isolines_provider: + self._isolines_provider = self.DEFAULT_PROVIDER + self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower() + if self._isolines_provider == self.HEREMAPS_PROVIDER: + self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) + self._heremaps_app_id = db_config.heremaps_isolines_app_id + self._heremaps_app_code = db_config.heremaps_isolines_app_code + if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_isolines_limit = True + else: + self._soft_isolines_limit = False + elif self._isolines_provider == self.MAPZEN_PROVIDER: + self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key + self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota self._soft_isolines_limit = False - self._heremaps_app_id = db_config.heremaps_isolines_app_id - self._heremaps_app_code = db_config.heremaps_isolines_app_code + self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) @property def service_type(self): - return 'here_isolines' + if self._isolines_provider == self.HEREMAPS_PROVIDER: + return 'here_isolines' + elif self._isolines_provider == self.MAPZEN_PROVIDER: + return 'mapzen_isolines' + + @property + def google_services_user(self): + return self._geocoder_provider == 'google' @property def isolines_quota(self): @@ -236,45 +207,23 @@ class IsolinesRoutingConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code - @property - def google_services_user(self): - return self._geocoder_type == self.GOOGLE_GEOCODER - - -class MapzenIsolinesRoutingConfig(ServiceConfig): - - PERIOD_END_DATE = 'period_end_date' - - def __init__(self, redis_connection, db_conn, username, orgname=None): - super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn, - username, orgname) - try: - self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key - self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota - self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) - except Exception as e: - raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e)) - - @property - def service_type(self): - return 'mapzen_isolines' - - @property - def isolines_quota(self): - return self._isolines_quota - - @property - def soft_isolines_limit(self): - return False - - @property - def period_end_date(self): - return self._period_end_date - @property def mapzen_matrix_api_key(self): return self._mapzen_matrix_api_key + @property + def mapzen_provider(self): + return self._isolines_provider == self.MAPZEN_PROVIDER + + @property + def heremaps_provider(self): + return self._isolines_provider == self.HEREMAPS_PROVIDER + + @property + def provider(self): + return self._isolines_provider + + class InternalGeocoderConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -308,7 +257,7 @@ class GeocoderConfig(ServiceConfig): GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', - 'geocoder_type', 'period_end_date', + 'geocoder_provider', 'period_end_date', 'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code', 'mapzen_geocoder_api_key', 'username', 'orgname'] NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] @@ -320,12 +269,13 @@ class GeocoderConfig(ServiceConfig): GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' MAPZEN_GEOCODER = 'mapzen' MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key' - GEOCODER_TYPE = 'geocoder_type' + GEOCODER_PROVIDER = 'geocoder_provider' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' + DEFAULT_PROVIDER = 'mapzen' def __init__(self, redis_connection, db_conn, username, orgname=None): super(GeocoderConfig, self).__init__(redis_connection, db_conn, @@ -335,21 +285,23 @@ class GeocoderConfig(ServiceConfig): self.__check_config(filtered_config) def __check_config(self, filtered_config): - if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if self._geocoder_provider == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \ not self.heremaps_app_id or not self.heremaps_app_code: raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") - elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + elif self._geocoder_provider == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") - elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: + elif self._geocoder_provider == self.MAPZEN_GEOCODER: if not self.mapzen_api_key: raise ConfigException("""Mapzen config is not setted up""") return True def __parse_config(self, filtered_config, db_config): - self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower() + if not self._geocoder_provider: + self._geocoder_provider = self.DEFAULT_PROVIDER self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) self._log_path = db_config.geocoder_log_path @@ -357,39 +309,39 @@ class GeocoderConfig(ServiceConfig): self._soft_geocoding_limit = True else: self._soft_geocoding_limit = False - if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if self._geocoder_provider == self.NOKIA_GEOCODER: self._heremaps_app_id = db_config.heremaps_geocoder_app_id self._heremaps_app_code = db_config.heremaps_geocoder_app_code self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit - elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + elif self._geocoder_provider == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] self._cost_per_hit = 0 - elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: + elif self._geocoder_provider == self.MAPZEN_GEOCODER: self._mapzen_api_key = db_config.mapzen_geocoder_api_key self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota self._cost_per_hit = 0 @property def service_type(self): - if self._geocoder_type == self.GOOGLE_GEOCODER: + if self._geocoder_provider == self.GOOGLE_GEOCODER: return 'geocoder_google' - elif self._geocoder_type == self.MAPZEN_GEOCODER: + elif self._geocoder_provider == self.MAPZEN_GEOCODER: return 'geocoder_mapzen' - else: + elif self._geocoder_provider == self.NOKIA_GEOCODER: return 'geocoder_here' @property def heremaps_geocoder(self): - return self._geocoder_type == self.NOKIA_GEOCODER + return self._geocoder_provider == self.NOKIA_GEOCODER @property def google_geocoder(self): - return self._geocoder_type == self.GOOGLE_GEOCODER + return self._geocoder_provider == self.GOOGLE_GEOCODER @property def mapzen_geocoder(self): - return self._geocoder_type == self.MAPZEN_GEOCODER + return self._geocoder_provider == self.MAPZEN_GEOCODER @property def google_client_id(self): @@ -570,6 +522,9 @@ class ServicesRedisConfig: OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota' OBS_GENERAL_QUOTA_KEY = 'obs_general_quota' PERIOD_END_DATE = 'period_end_date' + GEOCODER_PROVIDER_KEY = 'geocoder_provider' + ISOLINES_PROVIDER_KEY = 'isolines_provider' + ROUTING_PROVIDER_KEY = 'routing_provider' def __init__(self, redis_conn): self._redis_connection = redis_conn @@ -599,6 +554,15 @@ class ServicesRedisConfig: user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] if self.OBS_GENERAL_QUOTA_KEY in org_config: user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY] - user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] - user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] - user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] + if self.PERIOD_END_DATE in org_config: + user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + if self.GOOGLE_GEOCODER_CLIENT_ID in org_config: + user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] + if self.GOOGLE_GEOCODER_API_KEY in org_config: + user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] + if self.GEOCODER_PROVIDER_KEY in org_config: + user_config[self.GEOCODER_PROVIDER_KEY] = org_config[self.GEOCODER_PROVIDER_KEY] + if self.ISOLINES_PROVIDER_KEY in org_config: + user_config[self.ISOLINES_PROVIDER_KEY] = org_config[self.ISOLINES_PROVIDER_KEY] + if self.ROUTING_PROVIDER_KEY in org_config: + user_config[self.ROUTING_PROVIDER_KEY] = org_config[self.ROUTING_PROVIDER_KEY] diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 4834058..5b7feab 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.7.1', + version='0.7.2', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index fe6fd8f..06bd2d2 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -11,7 +11,9 @@ def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit) redis_conn.hset(user_redis_name, 'geocoding_quota', quota) redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota) - redis_conn.hset(user_redis_name, 'geocoder_type', service) + redis_conn.hset(user_redis_name, 'geocoder_provider', service) + redis_conn.hset(user_redis_name, 'isolines_provider', service) + redis_conn.hset(user_redis_name, 'routing_provider', service) redis_conn.hset(user_redis_name, 'period_end_date', end_date) if do_quota: redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota) diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index eca9d3e..7cc49dd 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis from cartodb_services.metrics import QuotaService -from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig +from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -109,6 +109,20 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=1500000) assert qs.check_user_quota() is False + def test_should_check_user_isolines_quota_correctly(self): + qs = self.__build_isolines_quota_service('test_user') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + + def test_should_check_org_isolines_quota_correctly(self): + qs = self.__build_isolines_quota_service('test_user', orgname='testorg') + qs.increment_success_service_use() + assert qs.check_user_quota() is True + qs.increment_success_service_use(amount=1500000) + assert qs.check_user_quota() is False + def test_should_check_user_obs_snapshot_quota_correctly(self): qs = self.__build_obs_snapshot_quota_service('test_user') qs.increment_success_service_use() @@ -149,7 +163,7 @@ class TestQuotaService(TestCase): username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) - def __build_routing_quota_service(self, username, service='routing_mapzen', + def __build_routing_quota_service(self, username, service='mapzen', orgname=None, soft_limit=False, quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, @@ -158,6 +172,15 @@ class TestQuotaService(TestCase): username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) + def __build_isolines_quota_service(self, username, service='mapzen', + orgname=None, soft_limit=False, + quota=100, end_date=datetime.today()): + self.__prepare_quota_service(username, quota, service, orgname, + soft_limit, 0, False, end_date) + isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock, + username, orgname) + return QuotaService(isolines_config, redis_connection=self.redis_conn) + def __build_obs_snapshot_quota_service(self, username, quota=100, service='obs_snapshot', orgname=None,