diff --git a/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql b/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql index 363fc39..c1b3c3b 100644 --- a/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql +++ b/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql @@ -3,74 +3,36 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.0'" to load this file. \quit -- HERE goes your code to upgrade/downgrade -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS TEXT AS $$ - import json - from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - - if service == 'routing': - round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN - api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys - elif service == 'geocoder': - round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN - api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys - elif service == 'isolines': - round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys - else: - return None - - round_robin = GD[round_robin_service] if round_robin_service in GD else 0 - - api_key = api_keys[round_robin] - - GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 - - return api_key -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], mode TEXT) RETURNS cdb_dataservices_server.simple_route AS $$ - import json - from cartodb_services.mapbox import MapboxRouting, MapboxRoutingResponse + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.metrics import QuotaService from cartodb_services.tools import Coordinate - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - quota_service = QuotaService(user_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) if not waypoints or len(waypoints) < 2: - logger.info("Empty origin or destination") - quota_service.increment_empty_service_use() + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] if len(waypoints) > 25: - logger.info("Too many waypoints (max 25)") - quota_service.increment_empty_service_use() + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] waypoint_coords = [] @@ -85,21 +47,21 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if resp and resp.shape: shape_linestring = polyline_to_linestring(resp.shape) if shape_linestring: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() return [shape_linestring, resp.length, int(round(resp.duration))] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to calculate Mapbox routing') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( @@ -252,52 +214,42 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + try: + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) - country_iso3 = None - if country: - country_iso3 = country_to_iso3(country) - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using mapbox') - finally: - quota_service.increment_total_service_use() + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; ---- cdb_geocode_namedplace_point(city_name text) @@ -360,101 +312,6 @@ RETURNS Geometry AS $$ return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - from cartodb_services.tools.country import country_to_iso3 - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) - - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapbox') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapzen import MapzenGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3, search_type='locality') - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapzen') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( username TEXT, orgname TEXT, @@ -463,29 +320,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -516,16 +365,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result.append([source, r, multipolygon]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(isolines)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isolines') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( @@ -536,31 +385,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -578,24 +418,24 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(result)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) return result else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isochrones') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) diff --git a/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql b/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql index 2a3f4cd..acabd05 100644 --- a/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql @@ -3,7 +3,6 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.29.0'" to load this file. \quit -- HERE goes your code to upgrade/downgrade -DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_apikey(username TEXT, orgname TEXT, service TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], mode TEXT); CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( diff --git a/server/extension/cdb_dataservices_server--0.30.0.sql b/server/extension/cdb_dataservices_server--0.30.0.sql index 43a6962..59c91d9 100644 --- a/server/extension/cdb_dataservices_server--0.30.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.0.sql @@ -13,39 +13,30 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi waypoints geometry(Point, 4326)[], mode TEXT) RETURNS cdb_dataservices_server.simple_route AS $$ - import json - from cartodb_services.mapbox import MapboxRouting, MapboxRoutingResponse + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.metrics import QuotaService from cartodb_services.tools import Coordinate - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - quota_service = QuotaService(user_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) if not waypoints or len(waypoints) < 2: - logger.info("Empty origin or destination") - quota_service.increment_empty_service_use() + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] if len(waypoints) > 25: - logger.info("Too many waypoints (max 25)") - quota_service.increment_empty_service_use() + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] waypoint_coords = [] @@ -60,21 +51,21 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if resp and resp.shape: shape_linestring = polyline_to_linestring(resp.shape) if shape_linestring: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() return [shape_linestring, resp.length, int(round(resp.duration))] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to calculate Mapbox routing') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( @@ -1774,34 +1765,6 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS TEXT AS $$ - import json - from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - - if service == 'routing': - round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN - api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys - elif service == 'geocoder': - round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN - api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys - elif service == 'isolines': - round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys - else: - return None - - round_robin = GD[round_robin_service] if round_robin_service in GD else 0 - - api_key = api_keys[round_robin] - - GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 - - return api_key -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN @@ -2094,52 +2057,42 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + try: + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) - country_iso3 = None - if country: - country_iso3 = country_to_iso3(country) - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using mapbox') - finally: - quota_service.increment_total_service_use() + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( @@ -2476,101 +2429,6 @@ RETURNS Geometry AS $$ return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - from cartodb_services.tools.country import country_to_iso3 - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) - - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapbox') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapzen import MapzenGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3, search_type='locality') - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapzen') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService @@ -3144,29 +3002,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -3197,16 +3047,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result.append([source, r, multipolygon]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(isolines)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isolines') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( @@ -3280,31 +3130,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -3322,24 +3163,24 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(result)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) return result else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isochrones') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index bce7c75..b57a3f9 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -10,39 +10,30 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi waypoints geometry(Point, 4326)[], mode TEXT) RETURNS cdb_dataservices_server.simple_route AS $$ - import json - from cartodb_services.mapbox import MapboxRouting, MapboxRoutingResponse + from cartodb_services.tools import ServiceManager + from cartodb_services.mapbox import MapboxRouting from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX - from cartodb_services.metrics import QuotaService from cartodb_services.tools import Coordinate - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.polyline import polyline_to_linestring + from cartodb_services.refactor.service.mapbox_routing_config import MapboxRoutingConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - - quota_service = QuotaService(user_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + service_manager = ServiceManager('routing', MapboxRoutingConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) if not waypoints or len(waypoints) < 2: - logger.info("Empty origin or destination") - quota_service.increment_empty_service_use() + service_manager.logger.info("Empty origin or destination") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] if len(waypoints) > 25: - logger.info("Too many waypoints (max 25)") - quota_service.increment_empty_service_use() + service_manager.logger.info("Too many waypoints (max 25)") + service_manager.quota_service.increment_empty_service_use() return [None, None, None] waypoint_coords = [] @@ -57,21 +48,21 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if resp and resp.shape: shape_linestring = polyline_to_linestring(resp.shape) if shape_linestring: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() return [shape_linestring, resp.length, int(round(resp.duration))] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [None, None, None] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to calculate Mapbox routing', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to calculate Mapbox routing') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints( diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index fa9e913..31d3331 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -117,32 +117,3 @@ RETURNS boolean AS $$ GD[cache_key] = obs_config return True $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS TEXT AS $$ - import json - from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - - if service == 'routing': - round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN - api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys - elif service == 'geocoder': - round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN - api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys - elif service == 'isolines': - round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN - api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys - else: - return None - - round_robin = GD[round_robin_service] if round_robin_service in GD else 0 - - api_key = api_keys[round_robin] - - GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 - - return api_key -$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index ed84e48..8fa38c8 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -185,50 +185,40 @@ $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools.country import country_to_iso3 + from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + try: + geocoder = MapboxGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) - with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) - country_iso3 = None - if country: - country_iso3 = country_to_iso3(country) - - coordinates = geocoder.geocode(searchtext=searchtext, city=city, - state_province=state_province, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode street point using mapbox') - finally: - quota_service.increment_total_service_use() + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + service_manager.quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + service_manager.quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapbox') + finally: + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 05ff3d9..55acc4e 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -58,101 +58,6 @@ RETURNS Geometry AS $$ return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point'] $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapbox import MapboxGeocoder - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - from cartodb_services.tools.country import country_to_iso3 - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapbox'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) - - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3) - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapbox') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.mapzen import MapzenGeocoder - from cartodb_services.tools.country import country_to_iso3 - from cartodb_services.metrics import QuotaService, metrics - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen'))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): - try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3, search_type='locality') - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapzen') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; - CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index b942327..5e5ba38 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -131,29 +131,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -184,16 +176,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ result.append([source, r, multipolygon]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(isolines)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isolines') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones( @@ -267,31 +259,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones( data_range integer[], options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json + from cartodb_services.tools import ServiceManager from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX from cartodb_services.tools import Coordinate from cartodb_services.tools.coordinates import coordinates_to_polygon - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig + from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + import cartodb_services + cartodb_services.init(plpy, GD) - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_isolines_routing_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') - - mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) - mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD) + service_manager.assert_within_limits() try: - client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) - mapbox_isolines = MapboxIsolines(client, logger) + client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params) + mapbox_isolines = MapboxIsolines(client, service_manager.logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -309,22 +292,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ for isochrone in resp: result_polygon = coordinates_to_polygon(isochrone.coordinates) if result_polygon: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() result.append([source, isochrone.duration, result_polygon]) else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() result.append([source, isochrone.duration, None]) - quota_service.increment_success_service_use() - quota_service.increment_isolines_service_use(len(result)) + service_manager.quota_service.increment_success_service_use() + service_manager.quota_service.increment_isolines_service_use(len(result)) return result else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return [] except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to get Mapbox isochrones') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py index 1c0e6cc..e2f9e8b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py @@ -1,6 +1,6 @@ -MAPBOX_ROUTING_APIKEY_ROUNDRROBIN = 'mapbox_routing_apikey_roundrrobin' -MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN = 'mapbox_geocoder_apikey_roundrrobin' -MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN = 'mapbox_isolines_apikey_roundrrobin' +MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin' +MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin' +MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin' TRANSPORT_MODE_TO_MAPBOX = { 'car': 'driving', diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_geocoder_config.py new file mode 100644 index 0000000..2f28643 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_geocoder_config.py @@ -0,0 +1,128 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.mapbox.types import MAPBOX_GEOCODER_APIKEY_ROUNDROBIN + + +class MapboxGeocoderConfig(object): + """ + Configuration needed to operate the Mapbox geocoder service. + """ + + def __init__(self, + geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + username, + organization, + service_params, + GD): + self._geocoding_quota = geocoding_quota + self._soft_geocoding_limit = soft_geocoding_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._mapbox_api_keys = mapbox_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'geocoder_mapbox' + + @property + def provider(self): + return 'mapbox' + + @property + def is_high_resolution(self): + return True + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def mapbox_api_key(self): + return round_robin(self._mapbox_api_keys, self._GD, + MAPBOX_GEOCODER_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + # TODO: for BW compat, remove + @property + def google_geocoder(self): + return False + + +class MapboxGeocoderConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + mapbox_server_conf = self._server_conf.get('mapbox_conf') + mapbox_api_keys = mapbox_server_conf['geocoder']['api_keys'] + mapbox_service_params = mapbox_server_conf['geocoder'].get('service', {}) + + geocoding_quota = self._get_quota() + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('geocoder_log_path', None) + + return MapboxGeocoderConfig(geocoding_quota, + soft_geocoding_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + self._username, + self._orgname, + mapbox_service_params, + self._GD) + + def _get_quota(self): + geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') + if geocoding_quota is '': + return 0 + + return int(geocoding_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py new file mode 100644 index 0000000..79baa29 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py @@ -0,0 +1,123 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.mapbox.types import MAPBOX_ISOLINES_APIKEY_ROUNDROBIN + + +class MapboxIsolinesConfig(object): + """ + Configuration needed to operate the Mapbox directions service. + """ + + def __init__(self, + isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + username, + organization, + service_params, + GD): + self._isolines_quota = isolines_quota + self._soft_isolines_limit = soft_isolines_limit + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._mapbox_api_keys = mapbox_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'mapbox_isolines' + + @property + def provider(self): + return 'mapbox' + + @property + def is_high_resolution(self): + return True + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return self._soft_isolines_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def mapbox_api_key(self): + return round_robin(self._mapbox_api_keys, self._GD, + MAPBOX_ISOLINES_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class MapboxIsolinesConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + mapbox_server_conf = self._server_conf.get('mapbox_conf') + mapbox_api_keys = mapbox_server_conf['matrix']['api_keys'] + mapbox_service_params = mapbox_server_conf['matrix'].get('service', {}) + + isolines_quota = self._get_quota() + soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('isolines_log_path', None) + + return MapboxIsolinesConfig(isolines_quota, + soft_isolines_limit, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + self._username, + self._orgname, + mapbox_service_params, + self._GD) + + def _get_quota(self): + isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota') + if isolines_quota is '': + return 0 + + return int(isolines_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_routing_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_routing_config.py new file mode 100644 index 0000000..79c907e --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_routing_config.py @@ -0,0 +1,131 @@ +from dateutil.parser import parse as date_parse +from cartodb_services.refactor.service.utils import round_robin +from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDROBIN + + +class MapboxRoutingConfig(object): + """ + Configuration needed to operate the Mapbox directions service. + """ + + def __init__(self, + routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + username, + organization, + service_params, + GD): + self._routing_quota = routing_quota + self._soft_routing_limit = soft_routing_limit + self._monthly_quota = monthly_quota + self._period_end_date = period_end_date + self._cost_per_hit = cost_per_hit + self._log_path = log_path + self._mapbox_api_keys = mapbox_api_keys + self._username = username + self._organization = organization + self._service_params = service_params + self._GD = GD + + @property + def service_type(self): + return 'routing_mapbox' + + @property + def provider(self): + return 'mapbox' + + @property + def is_high_resolution(self): + return True + + @property + def routing_quota(self): + return self._routing_quota + + @property + def soft_limit(self): + return self._soft_routing_limit + + @property + def monthly_quota(self): + return self._monthly_quota + + @property + def period_end_date(self): + return self._period_end_date + + @property + def cost_per_hit(self): + return self._cost_per_hit + + @property + def log_path(self): + return self._log_path + + @property + def mapbox_api_key(self): + return round_robin(self._mapbox_api_keys, self._GD, + MAPBOX_ROUTING_APIKEY_ROUNDROBIN) + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._organization + + @property + def service_params(self): + return self._service_params + + +class MapboxRoutingConfigBuilder(object): + + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._username = username + self._orgname = orgname + self._GD = GD + + def get(self): + mapbox_server_conf = self._server_conf.get('mapbox_conf') + mapbox_api_keys = mapbox_server_conf['routing']['api_keys'] + monthly_quota = mapbox_server_conf['routing']['monthly_quota'] + mapbox_service_params = mapbox_server_conf['routing'].get('service', {}) + + routing_quota = self._get_quota() + soft_routing_limit = self._user_conf.get('soft_mapzen_routing_limit').lower() == 'true' + cost_per_hit = 0 + period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') + period_end_date = date_parse(period_end_date_str) + + logger_conf = self._server_conf.get('logger_conf') + log_path = logger_conf.get('routing_log_path', None) + + return MapboxRoutingConfig(routing_quota, + soft_routing_limit, + monthly_quota, + period_end_date, + cost_per_hit, + log_path, + mapbox_api_keys, + self._username, + self._orgname, + mapbox_service_params, + self._GD) + + def _get_quota(self): + routing_quota = self._org_conf.get('mapzen_routing_quota') or self._user_conf.get('mapzen_routing_quota') + if routing_quota is '': + return 0 + + return int(routing_quota) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index af74e00..fdd4d0d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -86,7 +86,7 @@ class MapzenGeocoderConfig(object): class MapzenGeocoderConfigBuilder(object): - def __init__(self, server_conf, user_conf, org_conf, username, orgname): + def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD=None): self._server_conf = server_conf self._user_conf = user_conf self._org_conf = org_conf diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/utils.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/utils.py new file mode 100644 index 0000000..43da7d8 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/utils.py @@ -0,0 +1,6 @@ +def round_robin(elements, GD, key): + rr = GD[key] if key in GD else 0 + value = elements[rr] + GD[key] = rr + 1 if rr < len(elements) - 1 else 0 + + return value diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py index d493e40..51b8f5f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py @@ -5,10 +5,12 @@ from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + class RateLimitExceeded(Exception): def __str__(self): return repr('Rate limit exceeded') + class ServiceManagerBase: """ A Service manager collects the configuration needed to use a service, @@ -48,6 +50,8 @@ class ServiceManagerBase: @property def logger(self): return self.logger + + class ServiceManager(ServiceManagerBase): """ This service manager delegates the configuration parameter details, @@ -55,13 +59,13 @@ class ServiceManager(ServiceManagerBase): It uses the refactored configuration classes. """ - def __init__(self, service, config_builder, username, orgname): + def __init__(self, service, config_builder, username, orgname, GD=None): service_config = ServiceConfiguration(service, username, orgname) logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get() self.logger = Logger(logger_config) - self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get() + self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname, GD).get() rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get()