From 7dc87ad38e0995949cd4ff857c8b492b0c81ac88 Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 11 Jan 2018 10:18:01 +0100 Subject: [PATCH] Rotating API keys for Mapbox --- README.md | 2 +- ...db_dataservices_server--0.29.0--0.30.0.sql | 54 ++++++++++++++++-- ...db_dataservices_server--0.30.0--0.29.0.sql | 1 + .../cdb_dataservices_server--0.30.0.sql | 55 +++++++++++++++++-- server/extension/sql/100_routing_helper.sql | 5 +- server/extension/sql/15_config_helper.sql | 29 ++++++++++ server/extension/sql/20_geocode_street.sql | 5 +- server/extension/sql/50_namedplaces.sql | 5 +- server/extension/sql/80_isolines_helper.sql | 10 +++- .../test/expected/00_install_test.out | 2 +- server/extension/test/sql/00_install_test.sql | 2 +- .../cartodb_services/mapbox/types.py | 4 ++ .../cartodb_services/metrics/config.py | 40 +++++++------- .../cartodb_services/test/test_helper.py | 2 +- 14 files changed, 177 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index c3a3caf..2e0a669 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ SELECT CDB_Conf_SetConf( ```sql SELECT CDB_Conf_SetConf( 'mapbox_conf', - '{"routing": {"api_key": "your_api_key", "monthly_quota": 999999}, "geocoder": {"api_key": "your_api_key", "monthly_quota": 999999}, "matrix": {"api_key": "your_api_key", "monthly_quota": 1500000}}' + '{"routing": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "matrix": {"api_keys": ["your_api_key"], "monthly_quota": 1500000}}' ); ``` diff --git a/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql b/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql index 91a12e1..235b725 100644 --- a/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql +++ b/server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql @@ -2,6 +2,35 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.0'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS TEXT AS $$ + import json + from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + + if service == 'routing': + round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN + api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys + elif service == 'geocoder': + round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN + api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys + elif service == 'isolines': + round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys + else: + return None + + round_robin = GD[round_robin_service] if round_robin_service in GD else 0 + + api_key = api_keys[round_robin] + + GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 + + return api_key +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + -- HERE goes your code to upgrade/downgrade CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoints( username TEXT, @@ -28,8 +57,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + try: - client = MapboxRouting(user_routing_config.mapbox_api_key, logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) if not waypoints or len(waypoints) < 2: logger.info("Empty origin or destination") @@ -237,9 +269,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger, user_geocoder_config.mapbox_service_params) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) country_iso3 = None if country: @@ -344,9 +379,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) country_iso3 = None if country_name: @@ -442,8 +480,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: @@ -514,8 +555,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: diff --git a/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql b/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql index 36b935d..b5045f7 100644 --- a/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.0--0.29.0.sql @@ -3,6 +3,7 @@ \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.29.0'" to load this file. \quit -- HERE goes your code to upgrade/downgrade +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_apikey(username TEXT, orgname TEXT, service TEXT); DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapbox_route_with_waypoints(username TEXT, orgname TEXT, waypoints geometry(Point, 4326)[], mode TEXT); CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( diff --git a/server/extension/cdb_dataservices_server--0.30.0.sql b/server/extension/cdb_dataservices_server--0.30.0.sql index 3d8a632..955103c 100644 --- a/server/extension/cdb_dataservices_server--0.30.0.sql +++ b/server/extension/cdb_dataservices_server--0.30.0.sql @@ -32,8 +32,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + try: - client = MapboxRouting(user_routing_config.mapbox_api_key, logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) if not waypoints or len(waypoints) < 2: logger.info("Empty origin or destination") @@ -1770,6 +1773,36 @@ RETURNS boolean AS $$ GD[cache_key] = obs_config return True $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS TEXT AS $$ + import json + from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + + if service == 'routing': + round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN + api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys + elif service == 'geocoder': + round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN + api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys + elif service == 'isolines': + round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys + else: + return None + + round_robin = GD[round_robin_service] if round_robin_service in GD else 0 + + api_key = api_keys[round_robin] + + GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 + + return api_key +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN @@ -2079,9 +2112,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger, user_geocoder_config.mapbox_service_params) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) country_iso3 = None if country: @@ -2460,9 +2496,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) country_iso3 = None if country_name: @@ -3123,8 +3162,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: @@ -3258,8 +3300,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index d754300..bce7c75 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -29,8 +29,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'routing']) + try: - client = MapboxRouting(user_routing_config.mapbox_api_key, logger, user_routing_config.mapbox_service_params) + client = MapboxRouting(mapbox_apikey[0]['apikey'], logger, user_routing_config.mapbox_service_params) if not waypoints or len(waypoints) < 2: logger.info("Empty origin or destination") diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 31d3331..fa9e913 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -117,3 +117,32 @@ RETURNS boolean AS $$ GD[cache_key] = obs_config return True $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_apikey( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS TEXT AS $$ + import json + from cartodb_services.mapbox.types import MAPBOX_ROUTING_APIKEY_ROUNDRROBIN, MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN, MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + + if service == 'routing': + round_robin_service = MAPBOX_ROUTING_APIKEY_ROUNDRROBIN + api_keys = GD["user_routing_config_{0}".format(username)].mapbox_api_keys + elif service == 'geocoder': + round_robin_service = MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN + api_keys = GD["user_geocoder_config_{0}".format(username)].mapbox_api_keys + elif service == 'isolines': + round_robin_service = MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN + api_keys = GD["user_isolines_routing_config_{0}".format(username)].mapbox_matrix_api_keys + else: + return None + + round_robin = GD[round_robin_service] if round_robin_service in GD else 0 + + api_key = api_keys[round_robin] + + GD[round_robin_service] = round_robin + 1 if round_robin < len(api_keys) - 1 else 0 + + return api_key +$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 370e495..ed84e48 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -202,9 +202,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_mapbox_geocode_street_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger, user_geocoder_config.mapbox_service_params) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger, user_geocoder_config.mapbox_service_params) country_iso3 = None if country: diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 5d336c2..05ff3d9 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -77,9 +77,12 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'geocoder']) + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): try: - geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger) + geocoder = MapboxGeocoder(mapbox_apikey[0]['apikey'], logger) country_iso3 = None if country_name: diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 8f17c8a..b942327 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -148,8 +148,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: @@ -283,8 +286,11 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') + mapbox_apikey_query = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_apikey($1, $2, $3) as apikey;", ["text", "text", "text"]) + mapbox_apikey = plpy.execute(mapbox_apikey_query, [username, orgname, 'isolines']) + try: - client = MapboxMatrixClient(user_isolines_routing_config.mapbox_matrix_api_key, logger, user_isolines_routing_config.mapbox_matrix_service_params) + client = MapboxMatrixClient(mapbox_apikey[0]['apikey'], logger, user_isolines_routing_config.mapbox_matrix_service_params) mapbox_isolines = MapboxIsolines(client, logger) if source: diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 7266db8..f87ca05 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -31,7 +31,7 @@ SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing (1 row) -SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); +SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 9185567..9ffe2e1 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -13,7 +13,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localho SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); -SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}'); +SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}'); diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py index 4400203..1c0e6cc 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py @@ -1,3 +1,7 @@ +MAPBOX_ROUTING_APIKEY_ROUNDRROBIN = 'mapbox_routing_apikey_roundrrobin' +MAPBOX_GEOCODER_APIKEY_ROUNDRROBIN = 'mapbox_geocoder_apikey_roundrrobin' +MAPBOX_ISOLINES_APIKEY_ROUNDRROBIN = 'mapbox_isolines_apikey_roundrrobin' + TRANSPORT_MODE_TO_MAPBOX = { 'car': 'driving', 'walk': 'walking', diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 87d2a4b..aab4dd5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -149,7 +149,7 @@ class RoutingConfig(ServiceConfig): self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key self._mapzen_service_params = self._db_config.mapzen_routing_service_params - self._mapbox_api_key = self._db_config.mapbox_routing_api_key + self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys self._mapbox_service_params = self._db_config.mapbox_routing_service_params self._set_monthly_quota() self._set_soft_limit() @@ -183,8 +183,8 @@ class RoutingConfig(ServiceConfig): return self._routing_provider == self.MAPBOX_PROVIDER @property - def mapbox_api_key(self): - return self._mapbox_api_key + def mapbox_api_keys(self): + return self._mapbox_api_keys @property def mapbox_service_params(self): @@ -255,7 +255,7 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params elif self._isolines_provider == self.MAPBOX_PROVIDER: - self._mapbox_matrix_api_key = self._db_config.mapbox_matrix_api_key + self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params @@ -313,8 +313,8 @@ class IsolinesRoutingConfig(ServiceConfig): return self._isolines_provider == self.MAPZEN_PROVIDER @property - def mapbox_matrix_api_key(self): - return self._mapbox_matrix_api_key + def mapbox_matrix_api_keys(self): + return self._mapbox_matrix_api_keys @property def mapbox_matrix_service_params(self): @@ -385,7 +385,7 @@ class GeocoderConfig(ServiceConfig): MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key' GEOCODER_PROVIDER = 'geocoder_provider' MAPBOX_GEOCODER = 'mapbox' - MAPBOX_GEOCODER_API_KEY = 'mapbox_geocoder_api_key' + MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' USERNAME_KEY = 'username' @@ -413,7 +413,7 @@ class GeocoderConfig(ServiceConfig): if not self.mapzen_api_key: raise ConfigException("""Mapzen config is not set up""") elif self._geocoder_provider == self.MAPBOX_GEOCODER: - if not self.mapbox_api_key: + if not self.mapbox_api_keys: raise ConfigException("""Mapbox config is not set up""") return True @@ -447,7 +447,7 @@ class GeocoderConfig(ServiceConfig): self._cost_per_hit = 0 self._mapzen_service_params = db_config.mapzen_geocoder_service_params elif self._geocoder_provider == self.MAPBOX_GEOCODER: - self._mapbox_api_key = db_config.mapbox_geocoder_api_key + self._mapbox_api_keys = db_config.mapbox_geocoder_api_keys self._cost_per_hit = 0 self._mapbox_service_params = db_config.mapbox_geocoder_service_params @@ -522,8 +522,8 @@ class GeocoderConfig(ServiceConfig): return self._mapzen_service_params @property - def mapbox_api_key(self): - return self._mapbox_api_key + def mapbox_api_keys(self): + return self._mapbox_api_keys @property def mapbox_service_params(self): @@ -610,14 +610,14 @@ class ServicesDBConfig: raise ConfigException('Mapbox configuration missing') else: mapbox_conf = json.loads(mapbox_conf_json) - self._mapbox_matrix_api_key = mapbox_conf['matrix']['api_key'] + self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys'] self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota'] self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {}) self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {}) - self._mapbox_routing_api_key = mapbox_conf['routing']['api_key'] + self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys'] self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota'] self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {}) - self._mapbox_geocoder_api_key = mapbox_conf['geocoder']['api_key'] + self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys'] self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) @@ -715,8 +715,8 @@ class ServicesDBConfig: return self._mapzen_geocoder_service_params @property - def mapbox_matrix_api_key(self): - return self._mapbox_matrix_api_key + def mapbox_matrix_api_keys(self): + return self._mapbox_matrix_api_keys @property def mapbox_matrix_monthly_quota(self): @@ -731,8 +731,8 @@ class ServicesDBConfig: return self._mapbox_isochrones_service_params @property - def mapbox_routing_api_key(self): - return self._mapbox_routing_api_key + def mapbox_routing_api_keys(self): + return self._mapbox_routing_api_keys @property def mapbox_routing_monthly_quota(self): @@ -743,8 +743,8 @@ class ServicesDBConfig: return self._mapbox_routing_service_params @property - def mapbox_geocoder_api_key(self): - return self._mapbox_geocoder_api_key + def mapbox_geocoder_api_keys(self): + return self._mapbox_geocoder_api_keys @property def mapbox_geocoder_monthly_quota(self): diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index df0ac67..a8761e1 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -79,7 +79,7 @@ def increment_service_uses(redis_conn, username, orgname=None, def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) - plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])