cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add logger for the internal geocoder too

Browse Source
This commit is contained in:
Mario de Frutos
2016-03-08 16:08:56 +01:00
parent c293b55c3c
commit bd2e380225
12 changed files with 58 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql
View File

@@ -134,7 +134,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -165,7 +165,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -196,7 +196,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -227,7 +227,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -258,7 +258,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -289,7 +289,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -320,7 +320,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -351,7 +351,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -382,7 +382,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -413,7 +413,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -444,7 +444,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

22
server/extension/cdb_dataservices_server--0.5.1.sql
View File

@@ -227,7 +227,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -278,7 +278,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -310,7 +310,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -395,7 +395,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -427,7 +427,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -459,7 +459,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -558,7 +558,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -589,7 +589,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -620,7 +620,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -651,7 +651,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -777,7 +777,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

2
server/extension/sql/0.5.1/30_admin0.sql
View File

@@ -5,7 +5,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

4
server/extension/sql/0.5.1/40_admin1.sql
View File

@@ -6,7 +6,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -38,7 +38,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

6
server/extension/sql/0.5.1/50_namedplaces.sql
View File

@@ -6,7 +6,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -38,7 +38,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -70,7 +70,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

8
server/extension/sql/0.5.1/60_postalcodes.sql
View File

@@ -5,7 +5,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -36,7 +36,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -67,7 +67,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
@@ -98,7 +98,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

2
server/extension/sql/0.5.1/70_ips.sql
View File

@@ -5,7 +5,7 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname)
user_geocoder_config = InternalGeocoderConfig(redis_conn, plpy, username, orgname)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:

14
server/extension/test/0.5.0/expected/00_install_test.out
View File

@@ -19,7 +19,19 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
(1 row)
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
cdb_conf_setconf
------------------
(1 row)
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}');
cdb_conf_setconf
------------------
(1 row)
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
cdb_conf_setconf
------------------

4
server/extension/test/0.5.1/sql/00_install_test.sql
View File

@@ -12,8 +12,8 @@ CREATE EXTENSION cdb_dataservices_server;
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/var/log/postgresql/geocodings.log"}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)

12
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -69,8 +69,8 @@ class IsolinesRoutingConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, username,
orgname)
db_config = ServicesDBConfig(db_conn)
config = self.__get_user_config(username, orgname)
db_config = ServicesDBConfig(db_conn)
filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()}
self.__parse_config(filtered_config, db_config)
@@ -136,9 +136,11 @@ class IsolinesRoutingConfig(ServiceConfig):
class InternalGeocoderConfig(ServiceConfig):
def __init__(self, redis_connection, username, orgname=None):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(InternalGeocoderConfig, self).__init__(redis_connection,
username, orgname)
db_config = ServicesDBConfig(db_conn)
self._log_path = db_config.geocoder_log_path
@property
def service_type(self):
@@ -156,6 +158,10 @@ class InternalGeocoderConfig(ServiceConfig):
def geocoding_quota(self):
return None
@property
def log_path(self):
return self._log_path
class GeocoderConfig(ServiceConfig):
@@ -289,7 +295,7 @@ class GeocoderConfig(ServiceConfig):
@property
def cost_per_hit(self):
self._cost_per_hit
return self._cost_per_hit
@property
def log_path(self):

4
server/lib/python/cartodb_services/path
View File

@@ -1,4 +0,0 @@
{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.316618", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0}
{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.318872", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": "test_org", "used_credits": 0, "failed_rows": 0}
{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.322009", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0}
{"geocoder_type": "geocoder_here", "username": "test_user", "processed_rows": 1, "successful_rows": 1, "cost": null, "kind": "high-resolution", "success": true, "processable_rows": 1, "created_at": "2016-03-08T14:51:54.323941", "real_rows": 1, "batched": false, "cache_hits": 0, "organization": null, "used_credits": 0, "failed_rows": 0}

2
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -48,4 +48,4 @@ def _plpy_execute_side_effect(*args, **kwargs):
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing_app_key": "app_key"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "path"}'}]
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]