diff --git a/server/extension/test/0.5.1/sql/00_install_test.sql b/server/extension/test/0.5.1/sql/00_install_test.sql index 85c4fdb..1a296de 100644 --- a/server/extension/test/0.5.1/sql/00_install_test.sql +++ b/server/extension/test/0.5.1/sql/00_install_test.sql @@ -11,7 +11,9 @@ CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/var/log/postgresql/geocodings.log"}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 129e27e..d0670d7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -70,18 +70,16 @@ class IsolinesRoutingConfig(ServiceConfig): super(IsolinesRoutingConfig, self).__init__(redis_connection, username, orgname) db_config = ServicesDBConfig(db_conn) - config = self.__get_user_config(username, orgname, db_config) + config = self.__get_user_config(username, orgname) filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} - self.__parse_config(filtered_config) + self.__parse_config(filtered_config, db_config) - def __get_user_config(self, username, orgname, db_config): + def __get_user_config(self, username, orgname): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_APP_ID_KEY] = db_config.heremaps_app_id - user_config[self.NOKIA_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -96,7 +94,7 @@ class IsolinesRoutingConfig(ServiceConfig): user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] - def __parse_config(self, filtered_config): + def __parse_config(self, filtered_config, db_config): self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower() self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) @@ -104,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig): self._soft_isolines_limit = True else: self._soft_isolines_limit = False - self._heremaps_app_id = filtered_config[self.NOKIA_APP_ID_KEY] - self._heremaps_app_code = filtered_config[self.NOKIA_APP_CODE_KEY] + self._heremaps_app_id = db_config.heremaps_app_id + self._heremaps_app_code = db_config.heremaps_app_code @property def service_type(self): @@ -166,8 +164,7 @@ class GeocoderConfig(ServiceConfig): 'geocoder_type', 'period_end_date', 'heremaps_app_id', 'heremaps_app_code', 'username', 'orgname'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit', - 'heremaps_app_id', 'heremaps_app_code'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' @@ -180,25 +177,22 @@ class GeocoderConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' - LOG_PATH = '/var/log/postgresql/geocodings.log' def __init__(self, redis_connection, db_conn, username, orgname=None): super(GeocoderConfig, self).__init__(redis_connection, username, orgname) db_config = ServicesDBConfig(db_conn) - config = self.__get_user_config(username, orgname, db_config) + config = self.__get_user_config(username, orgname) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} + self.__parse_config(filtered_config, db_config) self.__check_config(filtered_config) - self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname, db_config): + def __get_user_config(self, username, orgname): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = db_config.heremaps_app_id - user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -219,28 +213,29 @@ class GeocoderConfig(ServiceConfig): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") - if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]: - raise ConfigException("""Nokia geocoder configuration is missing. Check it please""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") return True - def __parse_config(self, filtered_config): + def __parse_config(self, filtered_config, db_config): self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + self._log_path = db_config.geocoder_log_path if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: self._soft_geocoding_limit = False if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] - self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] + self._heremaps_app_id = db_config.heremaps_app_id + self._heremaps_app_code = db_config.heremaps_app_code + self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] + self._cost_per_hit = 0 @property def service_type(self): @@ -294,14 +289,11 @@ class GeocoderConfig(ServiceConfig): @property def cost_per_hit(self): - if self.heremaps_geocoder: - return 1 - else: - return 0 + self._cost_per_hit @property def log_path(self): - return self.LOG_PATH + return self._log_path class ServicesDBConfig: @@ -313,6 +305,7 @@ class ServicesDBConfig: def _build(self): self._get_here_config() self._get_mapzen_config() + self._get_logger_config() def _get_here_config(self): heremaps_conf_json = self._get_conf('heremaps_conf') @@ -322,6 +315,8 @@ class ServicesDBConfig: heremaps_conf = json.loads(heremaps_conf_json) self._heremaps_app_id = heremaps_conf['app_id'] self._heremaps_app_code = heremaps_conf['app_code'] + self._heremaps_geocoder_cost_per_hit = heremaps_conf[ + 'geocoder_cost_per_hit'] def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -331,6 +326,14 @@ class ServicesDBConfig: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] + def _get_logger_config(self): + logger_conf_json = self._get_conf('logger_conf') + if not logger_conf_json: + raise ConfigException('Logger configuration missing') + else: + logger_conf = json.loads(logger_conf_json) + self._geocoder_log_path = logger_conf['geocoder_log_path'] + def _get_conf(self, key): try: sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) @@ -347,6 +350,14 @@ class ServicesDBConfig: def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_geocoder_cost_per_hit(self): + return self._heremaps_geocoder_cost_per_hit + @property def mapzen_routing_app_key(self): return self._mapzen_routing_app_key + + @property + def geocoder_log_path(self): + return self._geocoder_log_path diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index b4b8d2b..29759e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -23,6 +23,7 @@ class Logger(object): def dump_to_file(self, data): with open(self._file_path, 'a') as logfile: json.dump(data, logfile) + logfile.write('\n') @abc.abstractproperty def log(self, **data): diff --git a/server/lib/python/cartodb_services/path b/server/lib/python/cartodb_services/path new file mode 100644 index 0000000..ce15f7f --- /dev/null +++ b/server/lib/python/cartodb_services/path @@ -0,0 +1,4 @@ +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.316618", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.318872", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.322009", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} +{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.323941", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0} diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index e732077..7aaebd3 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -43,7 +43,9 @@ def build_plpy_mock(empty=False): def _plpy_execute_side_effect(*args, **kwargs): - if args[0] == 'SELECT cartodb.CDB_Conf_GetConf(heremaps_conf) as conf': - return [{'conf': '{"app_id": "app_id", "app_code": "code"}'}] - elif args[0] == 'SELECT cartodb.CDB_Conf_GetConf(mapzen_conf) as conf': + if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": + return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": return [{'conf': '{"routing_app_key": "app_key"}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": + return [{'conf': '{"geocoder_log_path": "path"}'}]