From a046d3ce977cec08b7106fc266325f0c99d2700f Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Wed, 6 Jul 2016 12:40:31 +0200 Subject: [PATCH] Add Mapzen Matrix to config and metrics services --- server/extension/sql/15_config_helper.sql | 14 ++++++ server/extension/sql/80_isolines_helper.sql | 26 +++++------ server/extension/sql/90_isochrone.sql | 6 +-- .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 45 +++++++++++++++++++ .../cartodb_services/metrics/quota.py | 3 ++ 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 3d6fd72..118bbec 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -26,6 +26,20 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_mapzen_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import MapzenIsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = mapzen_isolines_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text) RETURNS boolean AS $$ cache_key = "user_internal_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 98b5091..874d8d1 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -71,20 +71,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.here.types import geo_polyline_to_multipolygon # TODO do we use the same types? redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] # -- Check the quota - #quota_service = QuotaService(user_isolines_routing_config, redis_conn) - #if not quota_service.check_user_quota(): - # plpy.error('You have reached the limit of your quota') + quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') try: - # --TODO: encapsulate or refactor this ugly code - mapzen_conf_str = plpy.execute("SELECT * FROM CDB_Conf_Getconf('mapzen_conf') AS mapzen_conf")[0]['mapzen_conf'] - mapzen_conf = json.loads(mapzen_conf_str) - - - client = MatrixClient(mapzen_conf['matrix']['api_key']) + client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key) mapzen_isolines = MapzenIsolines(client) if source: @@ -115,19 +110,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ multipolygon = plpy.execute(sql, 1)[0]['geom'] result.append([source, r, multipolygon]) - # --TODO take care of this quota/usage stuff - #quota_service.increment_success_service_use() - #quota_service.increment_isolines_service_use(len(resp)) + + quota_service.increment_success_service_use() + quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: import sys, traceback type_, value_, traceback_ = sys.exc_info() - #quota_service.increment_failed_service_use() + quota_service.increment_failed_service_use() error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) plpy.debug(traceback.format_tb(traceback_)) raise e #plpy.error(error_msg) finally: - pass - #quota_service.increment_total_service_use() + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 83af69b..8e24c94 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -26,11 +26,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)] type = 'isochrone' - # if we were to add a config check, it'll go here + # If we were to add a config check, it'll go here #if user_isolines_config.google_services_user: # plpy.error('This service is not available for google service users.') diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 9bba57c..78d09e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig +from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 340e5f6..e4d6597 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -241,6 +241,41 @@ class IsolinesRoutingConfig(ServiceConfig): return self._geocoder_type == self.GOOGLE_GEOCODER +class MapzenIsolinesRoutingConfig(ServiceConfig): + + PERIOD_END_DATE = 'period_end_date' + + def __init__(self, redis_connection, db_conn, username, orgname=None): + super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn, + username, orgname) + try: + self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key + self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + self._cost_per_hit = 0 + except Exception as e: + raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e)) + + @property + def service_type(self): + return 'mapzen_isolines' + + @property + def isolines_quota(self): + return self._isolines_quota + + @property + def soft_isolines_limit(self): + return False + + @property + def period_end_date(self): + return self._period_end_date + + @property + def mapzen_matrix_api_key(self): + return self._mapzen_matrix_api_key + class InternalGeocoderConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -438,6 +473,8 @@ class ServicesDBConfig: raise ConfigException('Mapzen configuration missing') else: mapzen_conf = json.loads(mapzen_conf_json) + self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] + self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] @@ -492,6 +529,14 @@ class ServicesDBConfig: def heremaps_geocoder_cost_per_hit(self): return self._heremaps_geocoder_cost_per_hit + @property + def mapzen_matrix_api_key(self): + return self._mapzen_matrix_api_key + + @property + def mapzen_matrix_monthly_quota(self): + return self._mapzen_matrix_quota + @property def mapzen_routing_api_key(self): return self._mapzen_routing_api_key diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index c627769..4496e3c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -70,6 +70,9 @@ class QuotaChecker: elif re.match('here_isolines', self._user_service_config.service_type) is not None: return self.__check_isolines_quota() + elif re.match('mapzen_isolines', + self._user_service_config.service_type) is not None: + return self.__check_isolines_quota() elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota()