From 49fd51f6da97ef5b6c1c97bd0a1f2f8d79ed5bde Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 2 Nov 2016 09:58:35 +0100 Subject: [PATCH] Added logger to metrics context manager --- .../sql/105_route_between_points.sql | 10 +++- .../sql/110_data_observatory_augmentation.sql | 40 +++++++------- .../sql/115_data_observatory_exploration.sql | 4 +- .../sql/120_data_observatory_geometries.sql | 14 ++--- server/extension/sql/20_geocode_street.sql | 6 ++- server/extension/sql/30_admin0.sql | 2 +- server/extension/sql/40_admin1.sql | 4 +- server/extension/sql/60_postalcodes.sql | 8 +-- server/extension/sql/70_ips.sql | 2 +- server/extension/sql/90_isochrone.sql | 5 +- .../cartodb_services/here/geocoder.py | 4 +- .../cartodb_services/here/routing.py | 4 +- .../cartodb_services/mapzen/matrix_client.py | 2 +- .../cartodb_services/mapzen/routing.py | 2 +- .../cartodb_services/metrics/log.py | 53 +++++++++++-------- 15 files changed, 92 insertions(+), 68 deletions(-) diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 2762c5f..217fc06 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -13,8 +13,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_route_with_point', user_routing_config): + with metrics('cdb_route_with_point', user_routing_config, logger): waypoints = [origin, destination] mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) @@ -36,8 +39,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_route_with_waypoints', user_routing_config): + with metrics('cdb_route_with_waypoints', user_routing_config, logger): mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index 4ccadfa..117ee9c 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -11,9 +11,9 @@ RETURNS text AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] - return user_obs_snapshot_config.connection_str + return user_obs_config.connection_str $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( @@ -42,16 +42,16 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) @@ -95,16 +95,16 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) @@ -151,16 +151,16 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) @@ -202,16 +202,16 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) @@ -272,7 +272,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getmeasure', user_obs_snapshot_config): + with metrics('obs_getmeasure', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) @@ -327,7 +327,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getcategory', user_obs_snapshot_config): + with metrics('obs_getcategory', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) @@ -384,7 +384,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getuscensusmeasure', user_obs_snapshot_config): + with metrics('obs_getuscensusmeasure', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) @@ -439,7 +439,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getuscensuscategory', user_obs_snapshot_config): + with metrics('obs_getuscensuscategory', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) @@ -494,7 +494,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpopulation', user_obs_snapshot_config): + with metrics('obs_getpopulation', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) @@ -549,7 +549,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getmeasurebyid', user_obs_snapshot_config): + with metrics('obs_getmeasurebyid', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index 05da6d8..9abe2b2 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -30,7 +30,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_search', user_obs_snapshot_config): + with metrics('obs_search', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) @@ -89,7 +89,7 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getavailableboundaries', user_obs_snapshot_config): + with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index bfc09fc..b69f318 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -32,7 +32,7 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundary', user_obs_snapshot_config): + with metrics('obs_getboundary', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) @@ -85,7 +85,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundaryid', user_obs_snapshot_config): + with metrics('obs_getboundaryid', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) @@ -138,7 +138,7 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundarybyid', user_obs_snapshot_config): + with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) @@ -193,7 +193,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config): + with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) @@ -255,7 +255,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config): + with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) @@ -315,7 +315,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpointsbygeometry', user_obs_snapshot_config): + with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) @@ -377,7 +377,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config): + with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index d1a4560..b4fbcd1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -2,12 +2,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_geocode_street_point', user_geocoder_config): + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): if user_geocoder_config.heremaps_geocoder: here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 7b06f97..593063a 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin0_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [country_name], 1) diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 29dbc02..d883e22 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -16,7 +16,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [admin1_name], 1) @@ -54,7 +54,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) rv = plpy.execute(plan, [admin1_name, country_name], 1) diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 8b9cb27..d7d610e 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) rv = plpy.execute(plan, [code], 1) @@ -50,7 +50,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) rv = plpy.execute(plan, [code, country], 1) @@ -86,7 +86,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [code], 1) @@ -122,7 +122,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) rv = plpy.execute(plan, [code, country], 1) diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 08d4559..2b452aa 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_ipaddress_point', user_geocoder_config): + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) rv = plpy.execute(plan, [ip], 1) diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 0464bc1..6f4186c 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -6,11 +6,14 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isochrone', user_isolines_config): + with metrics('cb_isochrone', user_isolines_config, logger): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index a221db9..d176a7d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -5,9 +5,10 @@ import json import requests from exceptions import * +from cartodb_services.metrics import Traceable -class HereMapsGeocoder: +class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -89,6 +90,7 @@ class HereMapsGeocoder: request_params.update(params) response = requests.get(self.host, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return json.loads(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 11856be..221a2ea 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -2,9 +2,10 @@ import requests import json from exceptions import WrongParams +from cartodb_services.metrics import Traceable -class HereMapsRoutingIsoline: +class HereMapsRoutingIsoline(Traceable): 'A Here Maps Routing wrapper for python' PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com' @@ -54,6 +55,7 @@ class HereMapsRoutingIsoline: parsed_options) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_isolines_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 8147be1..c10a912 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -46,7 +46,7 @@ class MatrixClient(Traceable): } response = requests.get(self.ONE_TO_MANY_URL, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) - self.add_response_data(response) + self.add_response_data(response, self._logger) if response.status_code != requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index 6c7c71b..d970d69 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -48,7 +48,7 @@ class MapzenRouting(Traceable): request_params = self.__parse_request_parameters(json_request_params) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) - self.add_response_data(response) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 3b94c19..67d02cf 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -7,7 +7,7 @@ from contextlib import contextmanager @contextmanager -def metrics(function, service_config): +def metrics(function, service_config, logger=None): try: start_time = time.time() yield @@ -15,16 +15,16 @@ def metrics(function, service_config): end_time = time.time() MetricsDataGatherer.add('function_name', function) MetricsDataGatherer.add('function_execution_time', (end_time - start_time)) - logger = MetricsServiceLoggerFactory.build(service_config) - if logger: + metrics_logger = MetricsServiceLoggerFactory.build(service_config, logger) + if metrics_logger: data = MetricsDataGatherer.get() - logger.log(data) + metrics_logger.log(data) MetricsDataGatherer.clean() class Traceable: - def add_response_data(self, response): + def add_response_data(self, response, logger=None): try: response_data = {} response_data['time'] = response.elapsed.total_seconds() @@ -37,7 +37,9 @@ class Traceable: MetricsDataGatherer.add('response', [response_data]) except BaseException as e: # We don't want to stop the job for some error here - plpy.warning(e) + if logger: + logger.error("Error trying to process response data for metrics", + exception=e) class MetricsDataGatherer: @@ -87,15 +89,15 @@ class MetricsDataGatherer: class MetricsServiceLoggerFactory: @classmethod - def build(self, service_config): + def build(self, service_config, logger=None): if re.search('^geocoder_*', service_config.service_type): - return MetricsGeocoderLogger(service_config) + return MetricsGeocoderLogger(service_config, logger) elif re.search('^routing_*', service_config.service_type): - return MetricsGenericLogger(service_config) + return MetricsGenericLogger(service_config, logger) elif re.search('_isolines$', service_config.service_type): - return MetricsIsolinesLogger(service_config) + return MetricsIsolinesLogger(service_config, logger) elif re.search('^obs_*', service_config.service_type): - return MetricsGenericLogger(service_config) + return MetricsGenericLogger(service_config, logger) else: return None @@ -103,15 +105,20 @@ class MetricsServiceLoggerFactory: class MetricsLogger(object): __metaclass__ = abc.ABCMeta - def __init__(self, service_config): + def __init__(self, service_config, logger): self._service_config = service_config + self._logger = logger def dump_to_file(self, data): - log_path = self.service_config.metrics_log_path - if log_path: - with open(log_path, 'a') as logfile: - json.dump(data, logfile) - logfile.write('\n') + try: + log_path = self.service_config.metrics_log_path + if log_path: + with open(log_path, 'a') as logfile: + json.dump(data, logfile) + logfile.write('\n') + except BaseException as e: + self._logger("Error dumping metrics to file {0}".format(log_path), + exception=e) def collect_data(self, data): return { @@ -141,8 +148,8 @@ class MetricsLogger(object): class MetricsGeocoderLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsGeocoderLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsGeocoderLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data) @@ -176,8 +183,8 @@ class MetricsGeocoderLogger(MetricsLogger): class MetricsGenericLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsGenericLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsGenericLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data) @@ -188,8 +195,8 @@ class MetricsGenericLogger(MetricsLogger): class MetricsIsolinesLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsIsolinesLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsIsolinesLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data)