cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added logger to metrics context manager

Browse Source
This commit is contained in:
Mario de Frutos
2016-11-02 09:58:35 +01:00
parent 708bee2912
commit 49fd51f6da
15 changed files with 92 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server/extension/sql/105_route_between_points.sql
View File

@@ -13,8 +13,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
with metrics('cdb_route_with_point', user_routing_config):
with metrics('cdb_route_with_point', user_routing_config, logger):
waypoints = [origin, destination]
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
@@ -36,8 +39,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
with metrics('cdb_route_with_waypoints', user_routing_config):
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]

40
server/extension/sql/110_data_observatory_augmentation.sql
View File

@@ -11,9 +11,9 @@ RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str
return user_obs_config.connection_str
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
@@ -42,16 +42,16 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config):
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
@@ -95,16 +95,16 @@ RETURNS SETOF JSON AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config):
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
@@ -151,16 +151,16 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config):
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
@@ -202,16 +202,16 @@ RETURNS SETOF JSON AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config):
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
@@ -272,7 +272,7 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getmeasure', user_obs_snapshot_config):
with metrics('obs_getmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
@@ -327,7 +327,7 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getcategory', user_obs_snapshot_config):
with metrics('obs_getcategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
@@ -384,7 +384,7 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getuscensusmeasure', user_obs_snapshot_config):
with metrics('obs_getuscensusmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
@@ -439,7 +439,7 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getuscensuscategory', user_obs_snapshot_config):
with metrics('obs_getuscensuscategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
@@ -494,7 +494,7 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpopulation', user_obs_snapshot_config):
with metrics('obs_getpopulation', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
@@ -549,7 +549,7 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getmeasurebyid', user_obs_snapshot_config):
with metrics('obs_getmeasurebyid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])

4
server/extension/sql/115_data_observatory_exploration.sql
View File

@@ -30,7 +30,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_search', user_obs_snapshot_config):
with metrics('obs_search', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
@@ -89,7 +89,7 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getavailableboundaries', user_obs_snapshot_config):
with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])

14
server/extension/sql/120_data_observatory_geometries.sql
View File

@@ -32,7 +32,7 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundary', user_obs_snapshot_config):
with metrics('obs_getboundary', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
@@ -85,7 +85,7 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundaryid', user_obs_snapshot_config):
with metrics('obs_getboundaryid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
@@ -138,7 +138,7 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundarybyid', user_obs_snapshot_config):
with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
@@ -193,7 +193,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config):
with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
@@ -255,7 +255,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config):
with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
@@ -315,7 +315,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpointsbygeometry', user_obs_snapshot_config):
with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
@@ -377,7 +377,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config):
with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])

6
server/extension/sql/20_geocode_street.sql
View File

@@ -2,12 +2,16 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
with metrics('cdb_geocode_street_point', user_geocoder_config):
with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']

2
server/extension/sql/30_admin0.sql
View File

@@ -14,7 +14,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config):
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)

4
server/extension/sql/40_admin1.sql
View File

@@ -16,7 +16,7 @@ RETURNS Geometry AS $$
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config):
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
@@ -54,7 +54,7 @@ RETURNS Geometry AS $$
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config):
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)

8
server/extension/sql/60_postalcodes.sql
View File

@@ -14,7 +14,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config):
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [code], 1)
@@ -50,7 +50,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config):
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
@@ -86,7 +86,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config):
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [code], 1)
@@ -122,7 +122,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_postalcode_point', user_geocoder_config):
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)

2
server/extension/sql/70_ips.sql
View File

@@ -14,7 +14,7 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_ipaddress_point', user_geocoder_config):
with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)

5
server/extension/sql/90_isochrone.sql
View File

@@ -6,11 +6,14 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_isolines_config.google_services_user:
raise Exception('This service is not available for google service users.')
with metrics('cb_isochrone', user_isolines_config):
with metrics('cb_isochrone', user_isolines_config, logger):
if user_isolines_config.heremaps_provider:
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])

4
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@@ -5,9 +5,10 @@ import json
import requests
from exceptions import *
from cartodb_services.metrics import Traceable
class HereMapsGeocoder:
class HereMapsGeocoder(Traceable):
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@@ -89,6 +90,7 @@ class HereMapsGeocoder:
request_params.update(params)
response = requests.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@@ -2,9 +2,10 @@ import requests
import json
from exceptions import WrongParams
from cartodb_services.metrics import Traceable
class HereMapsRoutingIsoline:
class HereMapsRoutingIsoline(Traceable):
'A Here Maps Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
@@ -54,6 +55,7 @@ class HereMapsRoutingIsoline:
parsed_options)
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text)
elif response.status_code == requests.codes.bad_request:

2
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
View File

@@ -46,7 +46,7 @@ class MatrixClient(Traceable):
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response)
self.add_response_data(response, self._logger)
if response.status_code != requests.codes.ok:
self._logger.error('Error trying to get matrix distance from mapzen',

2
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@@ -48,7 +48,7 @@ class MapzenRouting(Traceable):
request_params = self.__parse_request_parameters(json_request_params)
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response)
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_routing_response(response.text)
elif response.status_code == requests.codes.bad_request:

53
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@@ -7,7 +7,7 @@ from contextlib import contextmanager
@contextmanager
def metrics(function, service_config):
def metrics(function, service_config, logger=None):
try:
start_time = time.time()
yield
@@ -15,16 +15,16 @@ def metrics(function, service_config):
end_time = time.time()
MetricsDataGatherer.add('function_name', function)
MetricsDataGatherer.add('function_execution_time', (end_time - start_time))
logger = MetricsServiceLoggerFactory.build(service_config)
if logger:
metrics_logger = MetricsServiceLoggerFactory.build(service_config, logger)
if metrics_logger:
data = MetricsDataGatherer.get()
logger.log(data)
metrics_logger.log(data)
MetricsDataGatherer.clean()
class Traceable:
def add_response_data(self, response):
def add_response_data(self, response, logger=None):
try:
response_data = {}
response_data['time'] = response.elapsed.total_seconds()
@@ -37,7 +37,9 @@ class Traceable:
MetricsDataGatherer.add('response', [response_data])
except BaseException as e:
# We don't want to stop the job for some error here
plpy.warning(e)
if logger:
logger.error("Error trying to process response data for metrics",
exception=e)
class MetricsDataGatherer:
@@ -87,15 +89,15 @@ class MetricsDataGatherer:
class MetricsServiceLoggerFactory:
@classmethod
def build(self, service_config):
def build(self, service_config, logger=None):
if re.search('^geocoder_*', service_config.service_type):
return MetricsGeocoderLogger(service_config)
return MetricsGeocoderLogger(service_config, logger)
elif re.search('^routing_*', service_config.service_type):
return MetricsGenericLogger(service_config)
return MetricsGenericLogger(service_config, logger)
elif re.search('_isolines$', service_config.service_type):
return MetricsIsolinesLogger(service_config)
return MetricsIsolinesLogger(service_config, logger)
elif re.search('^obs_*', service_config.service_type):
return MetricsGenericLogger(service_config)
return MetricsGenericLogger(service_config, logger)
else:
return None
@@ -103,15 +105,20 @@ class MetricsServiceLoggerFactory:
class MetricsLogger(object):
__metaclass__ = abc.ABCMeta
def __init__(self, service_config):
def __init__(self, service_config, logger):
self._service_config = service_config
self._logger = logger
def dump_to_file(self, data):
log_path = self.service_config.metrics_log_path
if log_path:
with open(log_path, 'a') as logfile:
json.dump(data, logfile)
logfile.write('\n')
try:
log_path = self.service_config.metrics_log_path
if log_path:
with open(log_path, 'a') as logfile:
json.dump(data, logfile)
logfile.write('\n')
except BaseException as e:
self._logger("Error dumping metrics to file {0}".format(log_path),
exception=e)
def collect_data(self, data):
return {
@@ -141,8 +148,8 @@ class MetricsLogger(object):
class MetricsGeocoderLogger(MetricsLogger):
def __init__(self, service_config):
super(MetricsGeocoderLogger, self).__init__(service_config)
def __init__(self, service_config, logger):
super(MetricsGeocoderLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
@@ -176,8 +183,8 @@ class MetricsGeocoderLogger(MetricsLogger):
class MetricsGenericLogger(MetricsLogger):
def __init__(self, service_config):
super(MetricsGenericLogger, self).__init__(service_config)
def __init__(self, service_config, logger):
super(MetricsGenericLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
@@ -188,8 +195,8 @@ class MetricsGenericLogger(MetricsLogger):
class MetricsIsolinesLogger(MetricsLogger):
def __init__(self, service_config):
super(MetricsIsolinesLogger, self).__init__(service_config)
def __init__(self, service_config, logger):
super(MetricsIsolinesLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)