Remove info trace not needed in metrics

Release 0.17.0

NEWS.md

@@ -1,3 +1,8 @@
+October 21st, 2016
+==================
+* Version 0.9.2 of the python package
+    * mapzen routing quota now is configurable per user
+
 September 28, 2016
 ==========
 * Released version 0.8.1 of Python package cartodb\_services

doc/demographic_functions.md

@@ -1,6 +1,6 @@
 # Demographic Functions
 
-The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/carto-engine/dataservices-api/geocoding-functions/) documentation.
+The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
 
 _**Note:** The Demographic Snapshot functions are only available for the United States._
 

doc/geocoding_functions.md

@@ -290,8 +290,7 @@ Geocodes a postal code from a specified country into an IP address, displayed as
 
 Name | Type | Description
 --- | --- | ---
-`ip_address` | `text` | Postal code
-`country_name` | `text` | IPv4 or IPv6 address
+`ip_address` | `text` | IPv4 or IPv6 address
 
 #### Returns
 

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.16.0'
+default_version = '0.17.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/105_route_between_points.sql

@@ -7,15 +7,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
   options text[] DEFAULT ARRAY[]::text[],
   units text DEFAULT 'kilometers')
 RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
 
-  waypoints = [origin, destination]
-  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
-  result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
-  return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+  with metrics('cdb_route_with_point', user_routing_config, logger):
+    waypoints = [origin, destination]
+    mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+    result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
+    return [result[0]['shape'],result[0]['length'], result[0]['duration']]
 $$ LANGUAGE plpythonu;
 
 
@@ -27,12 +33,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
   options text[] DEFAULT ARRAY[]::text[],
   units text DEFAULT 'kilometers')
 RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
 
-  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
-  result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
-  return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger):
+    mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
+    result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
+    return [result[0]['shape'],result[0]['length'], result[0]['duration']]
 $$ LANGUAGE plpythonu;

server/extension/sql/110_data_observatory_augmentation.sql

@@ -11,9 +11,9 @@ RETURNS text AS $$
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
 
-  return user_obs_snapshot_config.connection_str
+  return user_obs_config.connection_str
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
@@ -34,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
   time_span TEXT DEFAULT NULL,
   geometry_level TEXT DEFAULT NULL)
 RETURNS json AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
   import json
@@ -41,31 +42,32 @@ RETURNS json AS $$
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  quota_service = QuotaService(user_obs_config, redis_conn)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to obs_get_demographic_snapshot')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
@@ -86,41 +88,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
   time_span TEXT DEFAULT NULL,
   geometry_level TEXT DEFAULT NULL)
 RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  quota_service = QuotaService(user_obs_config, redis_conn)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
-      if result:
-        resp = []
-        for element in result:
-          value = element['snapshot']
-          resp.append(value)
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to obs_get_demographic_snapshot')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
@@ -139,6 +143,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
   geom geometry(Geometry, 4326),
   geometry_level TEXT DEFAULT NULL)
 RETURNS json AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
   import json
@@ -146,31 +151,32 @@ RETURNS json AS $$
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  quota_service = QuotaService(user_obs_config, redis_conn)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['snapshot']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to obs_get_segment_snapshot')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_segment_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
@@ -189,41 +195,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
   geom geometry(Geometry, 4326),
   geometry_level TEXT DEFAULT NULL)
 RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
-  quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
+  quota_service = QuotaService(user_obs_config, redis_conn)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
-      if result:
-        resp = []
-        for element in result:
-          value = element['snapshot']
-          resp.append(value)
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetSegmentSnapshot')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetSegmentSnapshot')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
@@ -248,6 +256,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
   boundary_id TEXT DEFAULT NULL,
   time_span TEXT DEFAULT NULL)
 RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -263,22 +272,23 @@ RETURNS NUMERIC AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['measure']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetMeasure')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getmeasure', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['measure']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetMeasure')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
@@ -301,6 +311,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
   boundary_id TEXT DEFAULT NULL,
   time_span TEXT DEFAULT NULL)
 RETURNS TEXT AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -316,22 +327,23 @@ RETURNS TEXT AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['category']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetCategory')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getcategory', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['category']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetCategory')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
@@ -356,6 +368,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
   boundary_id TEXT DEFAULT NULL,
   time_span TEXT DEFAULT NULL)
 RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -371,22 +384,23 @@ RETURNS NUMERIC AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['census_measure']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetUSCensusMeasure')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getuscensusmeasure', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['census_measure']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetUSCensusMeasure')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
@@ -409,6 +423,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
   boundary_id TEXT DEFAULT NULL,
   time_span TEXT DEFAULT NULL)
 RETURNS TEXT AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -424,22 +439,23 @@ RETURNS TEXT AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['census_category']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetUSCensusCategory')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getuscensuscategory', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['census_category']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetUSCensusCategory')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
@@ -462,6 +478,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
   boundary_id TEXT DEFAULT NULL,
   time_span TEXT DEFAULT NULL)
 RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -477,22 +494,23 @@ RETURNS NUMERIC AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['population']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetPopulation')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getpopulation', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['population']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetPopulation')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
@@ -515,6 +533,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
   boundary_id TEXT,
   time_span TEXT DEFAULT NULL)
 RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -530,20 +549,21 @@ RETURNS NUMERIC AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['measure']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetMeasureById')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getmeasurebyid', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['measure']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetMeasureById')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;

server/extension/sql/115_data_observatory_exploration.sql

@@ -14,6 +14,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
   search_term TEXT,
   relevant_boundary TEXT DEFAULT NULL)
 RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -29,30 +30,31 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
-      if result:
-        resp = []
-        for element in result:
-          id = element['id']
-          description = element['description']
-          name = element['name']
-          aggregate = element['aggregate']
-          source = element['source']
-          resp.append([id, description, name, aggregate, source])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return [None, None, None, None, None]
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_Search')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_search', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
+        if result:
+          resp = []
+          for element in result:
+            id = element['id']
+            description = element['description']
+            name = element['name']
+            aggregate = element['aggregate']
+            source = element['source']
+            resp.append([id, description, name, aggregate, source])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return [None, None, None, None, None]
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_Search')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
@@ -71,6 +73,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
   geom geometry(Geometry, 4326),
   time_span TEXT DEFAULT NULL)
 RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -86,27 +89,28 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
-      if result:
-        resp = []
-        for element in result:
-          id = element['boundary_id']
-          description = element['description']
-          tspan = element['time_span']
-          tablename = element['tablename']
-          resp.append([id, description, tspan, tablename])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetMeasureById')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
+        if result:
+          resp = []
+          for element in result:
+            id = element['boundary_id']
+            description = element['description']
+            tspan = element['time_span']
+            tablename = element['tablename']
+            resp.append([id, description, tspan, tablename])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetMeasureById')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;

server/extension/sql/120_data_observatory_geometries.sql

@@ -16,6 +16,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
   boundary_id TEXT,
   time_span TEXT DEFAULT NULL)
 RETURNS geometry(Geometry, 4326) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -31,22 +32,23 @@ RETURNS geometry(Geometry, 4326) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['boundary']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetBoundary')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getboundary', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['boundary']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetBoundary')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
@@ -67,6 +69,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
   boundary_id TEXT,
   time_span TEXT DEFAULT NULL)
 RETURNS TEXT AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -82,22 +85,23 @@ RETURNS TEXT AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['boundary']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetBoundaryId')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getboundaryid', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['boundary']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetBoundaryId')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
@@ -118,6 +122,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
   boundary_id TEXT,
   time_span TEXT DEFAULT NULL)
 RETURNS geometry(Geometry, 4326) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -133,22 +138,23 @@ RETURNS geometry(Geometry, 4326) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
-      if result:
-        quota_service.increment_success_service_use()
-        return result[0]['boundary']
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetBoundaryById')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['boundary']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetBoundaryById')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
@@ -171,6 +177,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
   time_span TEXT DEFAULT NULL,
   overlap_type TEXT DEFAULT NULL)
 RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -186,27 +193,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
-      if result:
-        resp = []
-        for element in result:
-          the_geom = element['the_geom']
-          geom_refs = element['geom_refs']
-          resp.append([the_geom, geom_refs])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetBoundariesByGeometry')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
+        if result:
+          resp = []
+          for element in result:
+            the_geom = element['the_geom']
+            geom_refs = element['geom_refs']
+            resp.append([the_geom, geom_refs])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetBoundariesByGeometry')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
@@ -231,6 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
   time_span TEXT DEFAULT NULL,
   overlap_type TEXT DEFAULT NULL)
 RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -246,27 +255,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
-      if result:
-        resp = []
-        for element in result:
-          the_geom = element['the_geom']
-          geom_refs = element['geom_refs']
-          resp.append([the_geom, geom_refs])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
+        if result:
+          resp = []
+          for element in result:
+            the_geom = element['the_geom']
+            geom_refs = element['geom_refs']
+            resp.append([the_geom, geom_refs])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
@@ -289,6 +299,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
   time_span TEXT DEFAULT NULL,
   overlap_type TEXT DEFAULT NULL)
 RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -304,27 +315,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
-      if result:
-        resp = []
-        for element in result:
-          the_geom = element['the_geom']
-          geom_refs = element['geom_refs']
-          resp.append([the_geom, geom_refs])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return []
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetPointsByGeometry')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
+        if result:
+          resp = []
+          for element in result:
+            the_geom = element['the_geom']
+            geom_refs = element['geom_refs']
+            resp.append([the_geom, geom_refs])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetPointsByGeometry')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
@@ -349,6 +361,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
   time_span TEXT DEFAULT NULL,
   overlap_type TEXT DEFAULT NULL)
 RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.tools import Logger,LoggerConfig
 
@@ -364,25 +377,26 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-      obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
-      result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
-      if result:
-        resp = []
-        for element in result:
-          the_geom = element['the_geom']
-          geom_refs = element['geom_refs']
-          resp.append([the_geom, geom_refs])
-        quota_service.increment_success_service_use()
-        return resp
-      else:
-        quota_service.increment_empty_service_use()
-        return None
-  except BaseException as e:
-      import sys
-      quota_service.increment_failed_service_use()
-      logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
-      raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
-  finally:
-      quota_service.increment_total_service_use()
+  with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
+        if result:
+          resp = []
+          for element in result:
+            the_geom = element['the_geom']
+            geom_refs = element['geom_refs']
+            resp.append([the_geom, geom_refs])
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
+    finally:
+        quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;

server/extension/sql/20_geocode_street.sql

@@ -1,22 +1,28 @@
 -- Geocodes a street address given a searchtext and a state and/or country
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
 
-  if user_geocoder_config.heremaps_geocoder:
-    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
-    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
-  elif user_geocoder_config.google_geocoder:
-    google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
-    return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
-  elif user_geocoder_config.mapzen_geocoder:
-    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
-    return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
-  else:
-    raise Exception('Requested geocoder is not available')
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
 
 $$ LANGUAGE plpythonu;
 

server/extension/sql/30_admin0.sql

@@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
 RETURNS Geometry AS $$
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.metrics import metrics
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -13,23 +14,24 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
-    rv = plpy.execute(plan, [country_name], 1)
-    result = rv[0]["mypolygon"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode admin0 polygon')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
+      rv = plpy.execute(plan, [country_name], 1)
+      result = rv[0]["mypolygon"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode admin0 polygon')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 

server/extension/sql/40_admin1.sql

@@ -1,6 +1,7 @@
 ---- cdb_geocode_admin1_polygon(admin1_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -14,44 +15,11 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
-    rv = plpy.execute(plan, [admin1_name], 1)
-    result = rv[0]["mypolygon"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode admin1 polygon')
-  finally:
-    quota_service.increment_total_service_use()
-$$ LANGUAGE plpythonu;
 
----- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
-CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
-RETURNS Geometry AS $$
-    from cartodb_services.metrics import QuotaService
-    from cartodb_services.metrics import InternalGeocoderConfig
-    from cartodb_services.tools import Logger,LoggerConfig
-
-    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-    plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-    user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
-
-    plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
-    logger_config = GD["logger_config"]
-    logger = Logger(logger_config)
-    quota_service = QuotaService(user_geocoder_config, redis_conn)
+  with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
     try:
-      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
-      rv = plpy.execute(plan, [admin1_name, country_name], 1)
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
+      rv = plpy.execute(plan, [admin1_name], 1)
       result = rv[0]["mypolygon"]
       if result:
         quota_service.increment_success_service_use()
@@ -68,6 +36,44 @@ RETURNS Geometry AS $$
       quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
+---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+    from cartodb_services.metrics import metrics
+    from cartodb_services.metrics import QuotaService
+    from cartodb_services.metrics import InternalGeocoderConfig
+    from cartodb_services.tools import Logger,LoggerConfig
+
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+    plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+    user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+    plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+    logger_config = GD["logger_config"]
+    logger = Logger(logger_config)
+    quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+    with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
+      try:
+        plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
+        rv = plpy.execute(plan, [admin1_name, country_name], 1)
+        result = rv[0]["mypolygon"]
+        if result:
+          quota_service.increment_success_service_use()
+          return result
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+      except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to geocode admin1 polygon')
+      finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
 --------------------------------------------------------------------------------
 
 -- Implementation of the server extension

server/extension/sql/50_namedplaces.sql

@@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplac
 RETURNS Geometry AS $$
   from cartodb_services.mapzen import MapzenGeocoder
   from cartodb_services.mapzen.types import country_to_iso3
-  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import QuotaService, metrics
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -50,35 +50,36 @@ RETURNS Geometry AS $$
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
 
-  try:
-    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
-    country_iso3 = None
-    if country_name:
-      country_iso3 = country_to_iso3(country_name)
-    coordinates = geocoder.geocode(searchtext=city_name, city=None,
-                                   state_province=admin1_name,
-                                   country=country_iso3, search_type='locality')
-    if coordinates:
-      quota_service.increment_success_service_use()
-      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
-      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
-      return point['st_setsrid']
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode city point using mapzen')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
+    try:
+      geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+      country_iso3 = None
+      if country_name:
+        country_iso3 = country_to_iso3(country_name)
+      coordinates = geocoder.geocode(searchtext=city_name, city=None,
+                                    state_province=admin1_name,
+                                    country=country_iso3, search_type='locality')
+      if coordinates:
+        quota_service.increment_success_service_use()
+        plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+        point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+        return point['st_setsrid']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode city point using mapzen')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.metrics import QuotaService
-  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.metrics import InternalGeocoderConfig, metrics
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -90,30 +91,32 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    if admin1_name and country_name:
-      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
-      rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
-    elif country_name:
-      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
-      rv = plpy.execute(plan, [city_name, country_name], 1)
-    else:
-      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
-      rv = plpy.execute(plan, [city_name], 1)
-    result = rv[0]["mypoint"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode namedplace point')
-  finally:
-    quota_service.increment_total_service_use()
+
+  with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
+    try:
+      if admin1_name and country_name:
+        plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
+        rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
+      elif country_name:
+        plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
+        rv = plpy.execute(plan, [city_name, country_name], 1)
+      else:
+        plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
+        rv = plpy.execute(plan, [city_name], 1)
+      result = rv[0]["mypoint"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode namedplace point')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 --------------------------------------------------------------------------------

server/extension/sql/60_postalcodes.sql

@@ -1,5 +1,6 @@
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -13,27 +14,29 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
-    rv = plpy.execute(plan, [code], 1)
-    result = rv[0]["mypoint"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode postal code point')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
+      rv = plpy.execute(plan, [code], 1)
+      result = rv[0]["mypoint"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode postal code point')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -47,27 +50,29 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
-    rv = plpy.execute(plan, [code, country], 1)
-    result = rv[0]["mypoint"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode postal code point')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
+      rv = plpy.execute(plan, [code, country], 1)
+      result = rv[0]["mypoint"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode postal code point')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -81,27 +86,29 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
-    rv = plpy.execute(plan, [code], 1)
-    result = rv[0]["mypolygon"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode postal code polygon')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
+      rv = plpy.execute(plan, [code], 1)
+      result = rv[0]["mypolygon"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode postal code polygon')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -115,23 +122,24 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
-    rv = plpy.execute(plan, [code, country], 1)
-    result = rv[0]["mypolygon"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode postal code polygon')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
+      rv = plpy.execute(plan, [code, country], 1)
+      result = rv[0]["mypolygon"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode postal code polygon')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 --------------------------------------------------------------------------------

server/extension/sql/70_ips.sql

@@ -1,5 +1,6 @@
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
 RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
@@ -13,23 +14,24 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
-  try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
-    rv = plpy.execute(plan, [ip], 1)
-    result = rv[0]["mypoint"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
-  except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode postal code polygon')
-  finally:
-    quota_service.increment_total_service_use()
+  with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
+      rv = plpy.execute(plan, [ip], 1)
+      result = rv[0]["mypoint"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode postal code polygon')
+    finally:
+      quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 --------------------------------------------------------------------------------

server/extension/sql/90_isochrone.sql

@@ -1,21 +1,27 @@
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
 
   if user_isolines_config.google_services_user:
     raise Exception('This service is not available for google service users.')
 
-  if user_isolines_config.heremaps_provider:
-    here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-    return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
-  elif user_isolines_config.mapzen_provider:
-    mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-    return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
-  else:
-    raise Exception('Requested isolines provider is not available')
+  with metrics('cb_isochrone', user_isolines_config, logger):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
 $$ LANGUAGE plpythonu;
 
 -- heremaps isochrone

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -5,15 +5,18 @@ import json
 import requests
 
 from exceptions import *
+from cartodb_services.metrics import Traceable
 
 
-class HereMapsGeocoder:
+class HereMapsGeocoder(Traceable):
     'A Here Maps Geocoder wrapper for python'
 
     PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
     STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
     DEFAULT_MAXRESULTS = 1
     DEFAULT_GEN = 9
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ADDRESS_PARAMS = [
         'city',
@@ -85,7 +88,9 @@ class HereMapsGeocoder:
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params)
+        response = requests.get(self.host, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
+        self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:
             return json.loads(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -2,14 +2,17 @@ import requests
 import json
 
 from exceptions import WrongParams
+from cartodb_services.metrics import Traceable
 
 
-class HereMapsRoutingIsoline:
+class HereMapsRoutingIsoline(Traceable):
     'A Here Maps Routing wrapper for python'
 
     PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
     STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -50,7 +53,9 @@ class HereMapsRoutingIsoline:
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
-        response = requests.get(self._url, params=request_params)
+        response = requests.get(self._url, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
+        self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:
             return self.__parse_isolines_response(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -5,12 +5,15 @@ import re
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
+from cartodb_services.metrics import Traceable
 
 
-class MapzenGeocoder:
+class MapzenGeocoder(Traceable):
     'A Mapzen Geocoder wrapper for python'
 
     BASE_URL = 'https://search.mapzen.com/v1/search'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -24,7 +27,9 @@ class MapzenGeocoder:
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params)
+            response = requests.get(self._url, params=request_params,
+                                    timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
+            self.add_response_data(response, self._logger)
             if response.status_code == requests.codes.ok:
                 return self.__parse_response(response.text)
             elif response.status_code == requests.codes.bad_request:
@@ -41,6 +46,12 @@ class MapzenGeocoder:
                                       "state_province": state_province})
                 raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
                                        response)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            self._logger.error('Timeout connecting to Mapzen geocoding server')
+            raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
+                                    None)
         except requests.ConnectionError as e:
             # Don't raise the exception to continue with the geocoding job
             self._logger.error('Error connecting to Mapzen geocoding server',

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -86,7 +86,7 @@ class MapzenIsolines:
     def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
 
         # NOTE: not for production
-        self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
+        # self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
 
         # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
         # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
@@ -105,14 +105,16 @@ class MapzenIsolines:
 
             response = self._matrix_client.one_to_many([origin] + location_estimates,  costing_model)
             costs = [None] * self.NUMBER_OF_ANGLES
+            if not response:
+                # In case the matrix client doesn't return any data
+                break
+
             for idx, c in enumerate(response['one_to_many'][0][1:]):
                 if c[cost_variable]:
                     costs[idx] = c[cost_variable]*unit_factor
                 else:
                     costs[idx] = isorange
 
-            # self._logger.debug('i = %d, costs = %s' % (i, costs))
-
             errors = [(cost - isorange) / float(isorange) for cost in costs]
             max_abs_error = max([abs(e) for e in errors])
             if max_abs_error <= self.TOLERANCE:

server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py

@@ -2,9 +2,10 @@ import requests
 import json
 from qps import qps_retry
 from exceptions import ServiceException
+from cartodb_services.metrics import Traceable
 
 
-class MatrixClient:
+class MatrixClient(Traceable):
 
     """
     A minimal client for Mapzen Time-Distance Matrix Service
@@ -19,6 +20,8 @@ class MatrixClient:
     """
 
     ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     def __init__(self, matrix_key, logger):
         self._matrix_key = matrix_key
@@ -41,9 +44,11 @@ class MatrixClient:
             'costing': costing,
             'api_key': self._matrix_key
         }
-        response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
+        response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
+        self.add_response_data(response, self._logger)
 
-        if not requests.codes.ok:
+        if response.status_code != requests.codes.ok:
             self._logger.error('Error trying to get matrix distance from mapzen',
                                data={"response_status": response.status_code,
                                      "response_reason": response.reason,
@@ -52,6 +57,22 @@ class MatrixClient:
                                      "response_headers": response.headers,
                                      "locations": locations,
                                      "costing": costing})
-            raise ServiceException("Error trying to get matrix distance from mapzen", response)
+            # In case 4xx error we return empty because the error comes from
+            # the provided info by the user and we don't want to top the
+            # isolines generation
+            if response.status_code == requests.codes.bad_request:
+                return {}
+            elif response.status_code == 504:
+                # Due to some unsolved problems in the Mapzen Matrix API we're
+                # getting randomly 504, probably timeouts. To avoid raise an
+                # exception in all the jobs, for now we're going to return
+                # empty in that case
+                return {}
+            else:
+                raise ServiceException("Error trying to get matrix distance from mapzen", response)
 
-        return response.json()
+        # response could return with empty json
+        try:
+            return response.json()
+        except:
+            return {}

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -5,12 +5,15 @@ import re
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
+from cartodb_services.metrics import MetricsDataGatherer, Traceable
 
 
-class MapzenRouting:
+class MapzenRouting(Traceable):
     'A Mapzen Routing wrapper for python'
 
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -43,7 +46,9 @@ class MapzenRouting:
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params)
+        response = requests.get(self._url, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
+        self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:
             return self.__parse_routing_response(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,3 +1,4 @@
 from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService
+from log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -15,6 +15,7 @@ class ServiceConfig(object):
         self._username = username
         self._orgname = orgname
         self._db_config = ServicesDBConfig(db_conn, username, orgname)
+        self._metrics_log_path = self.__get_metrics_log_path()
         self._environment = self._db_config._server_environment
         if redis_connection:
             self._redis_config = ServicesRedisConfig(redis_connection).build(
@@ -38,9 +39,20 @@ class ServiceConfig(object):
     def environment(self):
         return self._environment
 
+    @property
+    def metrics_log_path(self):
+        return self._metrics_log_path
+
+    def __get_metrics_log_path(self):
+        if self.METRICS_LOG_KEY:
+            return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None)
+        else:
+            return None
 
 class DataObservatoryConfig(ServiceConfig):
 
+    METRICS_LOG_KEY = 'do_log_path'
+
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
                                             username, orgname)
@@ -61,6 +73,10 @@ class DataObservatoryConfig(ServiceConfig):
     def connection_str(self):
         return self._connection_str
 
+    @property
+    def provider(self):
+        return 'data observatory'
+
 
 class ObservatorySnapshotConfig(DataObservatoryConfig):
 
@@ -116,6 +132,9 @@ class RoutingConfig(ServiceConfig):
     ROUTING_PROVIDER_KEY = 'routing_provider'
     MAPZEN_PROVIDER = 'mapzen'
     DEFAULT_PROVIDER = 'mapzen'
+    QUOTA_KEY = 'mapzen_routing_quota'
+    SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
+    METRICS_LOG_KEY = 'routing_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         super(RoutingConfig, self).__init__(redis_connection, db_conn,
@@ -124,7 +143,8 @@ class RoutingConfig(ServiceConfig):
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
         self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
+        self._set_monthly_quota()
+        self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
     @property
@@ -132,6 +152,10 @@ class RoutingConfig(ServiceConfig):
         if self._routing_provider == self.MAPZEN_PROVIDER:
             return 'routing_mapzen'
 
+    @property
+    def provider(self):
+        return self._routing_provider
+
     @property
     def mapzen_api_key(self):
         return self._mapzen_api_key
@@ -144,6 +168,26 @@ class RoutingConfig(ServiceConfig):
     def period_end_date(self):
         return self._period_end_date
 
+    @property
+    def soft_limit(self):
+        return self._soft_limit
+
+    def _set_monthly_quota(self):
+        self._monthly_quota = self._get_effective_monthly_quota()
+
+    def _get_effective_monthly_quota(self):
+        quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
+        if quota_from_redis and quota_from_redis <> '':
+            return int(quota_from_redis)
+        else:
+            return self._db_config.mapzen_routing_monthly_quota
+
+    def _set_soft_limit(self):
+        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
+            self._soft_limit = True
+        else:
+            self._soft_limit = False
+
 
 class IsolinesRoutingConfig(ServiceConfig):
 
@@ -159,6 +203,7 @@ class IsolinesRoutingConfig(ServiceConfig):
     MAPZEN_PROVIDER = 'mapzen'
     HEREMAPS_PROVIDER = 'heremaps'
     DEFAULT_PROVIDER = 'heremaps'
+    METRICS_LOG_KEY = 'isolines_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
@@ -235,11 +280,12 @@ class IsolinesRoutingConfig(ServiceConfig):
 
 class InternalGeocoderConfig(ServiceConfig):
 
+    METRICS_LOG_KEY = 'geocoder_log_path'
+
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         # For now, internal geocoder doesn't use the redis config
         super(InternalGeocoderConfig, self).__init__(None, db_conn,
                                                      username, orgname)
-        self._log_path = self._db_config.geocoder_log_path
 
     @property
     def service_type(self):
@@ -258,8 +304,8 @@ class InternalGeocoderConfig(ServiceConfig):
         return None
 
     @property
-    def log_path(self):
-        return self._log_path
+    def provider(self):
+        return 'internal'
 
 
 class GeocoderConfig(ServiceConfig):
@@ -285,6 +331,7 @@ class GeocoderConfig(ServiceConfig):
     ORGNAME_KEY = 'orgname'
     PERIOD_END_DATE = 'period_end_date'
     DEFAULT_PROVIDER = 'mapzen'
+    METRICS_LOG_KEY = 'geocoder_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
         super(GeocoderConfig, self).__init__(redis_connection, db_conn,
@@ -316,7 +363,6 @@ class GeocoderConfig(ServiceConfig):
             self._geocoder_provider = self.DEFAULT_PROVIDER
         self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
-        self._log_path = db_config.geocoder_log_path
         if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_geocoding_limit = True
         else:
@@ -399,8 +445,8 @@ class GeocoderConfig(ServiceConfig):
         return self._cost_per_hit
 
     @property
-    def log_path(self):
-        return self._log_path
+    def provider(self):
+        return self._geocoder_provider
 
 
 class ServicesDBConfig:
@@ -415,7 +461,6 @@ class ServicesDBConfig:
         self._get_server_config()
         self._get_here_config()
         self._get_mapzen_config()
-        self._get_logger_config()
         self._get_data_observatory_config()
 
     def _get_server_config(self):
@@ -468,13 +513,6 @@ class ServicesDBConfig:
             else:
                 self._data_observatory_connection_str = do_conf['connection']['production']
 
-    def _get_logger_config(self):
-        logger_conf_json = self._get_conf('logger_conf')
-        if not logger_conf_json:
-            raise ConfigException('Logger configuration missing')
-        else:
-            logger_conf = json.loads(logger_conf_json)
-            self._geocoder_log_path = logger_conf['geocoder_log_path']
 
     def _get_conf(self, key):
         try:
@@ -482,7 +520,7 @@ class ServicesDBConfig:
             conf = self._db_conn.execute(sql, 1)
             return conf[0]['conf']
         except Exception as e:
-            raise ConfigException("Malformed config for {0}: {1}".format(key, e))
+            raise ConfigException("Error trying to get config for {0}: {1}".format(key, e))
 
     @property
     def server_environment(self):
@@ -532,14 +570,18 @@ class ServicesDBConfig:
     def mapzen_geocoder_monthly_quota(self):
         return self._mapzen_geocoder_quota
 
-    @property
-    def geocoder_log_path(self):
-        return self._geocoder_log_path
-
     @property
     def data_observatory_connection_str(self):
         return self._data_observatory_connection_str
 
+    @property
+    def logger_config(self):
+        logger_conf_json = self._get_conf('logger_conf')
+        if not logger_conf_json:
+            raise ConfigException('Logger configuration missing')
+        else:
+            return json.loads(logger_conf_json)
+
 
 class ServicesRedisConfig:
 
@@ -547,6 +589,7 @@ class ServicesRedisConfig:
     GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
+    ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
     OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
     OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
@@ -585,8 +628,12 @@ class ServicesRedisConfig:
         if not org_config:
             raise ConfigException("""There is no organization config available. Please check your configuration.'""")
         else:
-            user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
-            user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+            if self.QUOTA_KEY in org_config:
+                user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
+            if self.ISOLINES_QUOTA_KEY in org_config:
+                user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+            if self.ROUTING_QUOTA_KEY in org_config:
+                user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
             if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
                 user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
             if self.OBS_GENERAL_QUOTA_KEY in org_config:

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -1,15 +1,130 @@
-from datetime import datetime
 import abc
 import json
 import re
+import time
+import uuid
+import plpy
+from datetime import datetime
+from contextlib import contextmanager
+from urlparse import urlparse
 
 
-class MetricsLoggerFactory:
+@contextmanager
+def metrics(function, service_config, logger=None):
+    try:
+        start_time = time.time()
+        yield
+    finally:
+        end_time = time.time()
+        MetricsDataGatherer.add('uuid', str(uuid.uuid1()))
+        MetricsDataGatherer.add('function_name', function)
+        MetricsDataGatherer.add('function_execution_time', (end_time - start_time))
+        metrics_logger = MetricsServiceLoggerFactory.build(service_config,
+                                                           logger)
+        if metrics_logger:
+            data = MetricsDataGatherer.get()
+            metrics_logger.log(data)
+        MetricsDataGatherer.clean()
+
+
+class Traceable:
+    """
+    Module to add metrics traceability, for example to get response object
+    in order to add to the metrics dump
+    """
+    def add_response_data(self, response, logger=None):
+        try:
+            response_data = {}
+            response_data['type'] = "request"
+            response_data['date'] = datetime.now().isoformat()
+            response_data['elapsed_time'] = response.elapsed.total_seconds()
+            response_data['code'] = response.status_code
+            response_data['message'] = response.reason
+            response_data['url'] = self._parse_response_url(response.url)
+            stored_data = MetricsDataGatherer.get_element('response')
+            if stored_data:
+                stored_data.append(response_data)
+            else:
+                MetricsDataGatherer.add('response', [response_data])
+        except BaseException as e:
+            # We don't want to stop the job for some error processing response
+            if logger:
+                logger.error("Error trying to process response metricd data",
+                             exception=e)
+
+    def _parse_response_url(self, url):
+        u = urlparse(url)
+        return "{0}://{1}{2}".format(u.scheme, u.netloc, u.path)
+
+
+class MetricsDataGatherer:
+    """
+    Metrics gatherer used as a singleton. The intend is to use it as a global
+    storage for the metrics along the function request.
+    """
+
+    class __MetricsDataGatherer:
+        def __init__(self):
+            self.data = {}
+
+        def add(self, key, value):
+            self.data[key] = value
+
+        def get(self):
+            return self.data
+
+        def get_element(self, key):
+            return self.data.get(key, None)
+
+        def clean(self):
+            self.data = {}
+
+
+    #  We use pgbouncer so we need to have multiples instances per request id
+    __instance = {}
 
     @classmethod
-    def build(self, service_config):
-        if re.match('geocoder_*', service_config.service_type):
-            return MetricsGeocoderLogger(service_config)
+    def add(self, key, value):
+        MetricsDataGatherer.instance().add(key, value)
+
+    @classmethod
+    def get(self):
+        return MetricsDataGatherer.instance().get()
+
+    @classmethod
+    def get_element(self, key):
+        return MetricsDataGatherer.instance().get_element(key)
+
+    @classmethod
+    def clean(self):
+        MetricsDataGatherer.instance().clean()
+
+    @classmethod
+    def instance(self):
+        txid = MetricsDataGatherer._get_txid()
+        if txid not in MetricsDataGatherer.__instance:
+            MetricsDataGatherer.__instance[txid] = MetricsDataGatherer.__MetricsDataGatherer()
+
+        return MetricsDataGatherer.__instance[txid]
+
+    @classmethod
+    def _get_txid(self):
+        result = plpy.execute('select txid_current() as txid')
+        return result[0]['txid']
+
+
+class MetricsServiceLoggerFactory:
+
+    @classmethod
+    def build(self, service_config, logger=None):
+        if re.search('^geocoder_*', service_config.service_type):
+            return MetricsGeocoderLogger(service_config, logger)
+        elif re.search('^routing_*', service_config.service_type):
+            return MetricsGenericLogger(service_config, logger)
+        elif re.search('_isolines$', service_config.service_type):
+            return MetricsIsolinesLogger(service_config, logger)
+        elif re.search('^obs_*', service_config.service_type):
+            return MetricsGenericLogger(service_config, logger)
         else:
             return None
 
@@ -17,58 +132,119 @@ class MetricsLoggerFactory:
 class MetricsLogger(object):
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, file_path):
-        self._file_path = file_path
+    def __init__(self, service_config, logger):
+        self._service_config = service_config
+        self._logger = logger
 
     def dump_to_file(self, data):
-        with open(self._file_path, 'a') as logfile:
-            json.dump(data, logfile)
-            logfile.write('\n')
+        try:
+            log_path = self.service_config.metrics_log_path
+            response_data = data.pop('response', [])
+            uuid = data.get('uuid')
+            if log_path:
+                with open(log_path, 'a') as logfile:
+                    self._dump_response_to_file(uuid, response_data, logfile)
+                    json.dump(data, logfile)
+                    logfile.write('\n')
+        except BaseException as e:
+            self._logger("Error dumping metrics to file {0}".format(log_path),
+                         exception=e)
+
+    def collect_data(self, data):
+        return {
+            "uuid": data.get('uuid', uuid.uuid1()),
+            "type": 'function',
+            "function_name": data.get('function_name', None),
+            "function_execution_time": data.get('function_execution_time',
+                                                None),
+            "service": self._service_config.service_type,
+            "processable_rows": 1,
+            "success": data.get('success', False),
+            "successful_rows": data.get('successful_rows', 0),
+            "failed_rows": data.get('failed_rows', 0),
+            "empty_rows": data.get('empty_rows', 0),
+            "created_at": datetime.now().isoformat(),
+            "provider": self._service_config.provider,
+            "username": self._service_config.username,
+            "organization": self._service_config.organization,
+            "response": data.get('response', [])
+        }
+
+    def _dump_response_to_file(self, uuid, response_data, log_file):
+        for r in response_data:
+            r['uuid'] = uuid
+            json.dump(r, log_file)
+            log_file.write('\n')
+
+    @property
+    def service_config(self):
+        return self._service_config
 
     @abc.abstractproperty
-    def log(self, **data):
+    def log(self, data):
         raise NotImplementedError('log method must be defined')
 
 
 class MetricsGeocoderLogger(MetricsLogger):
 
-    def __init__(self, service_config):
-        super(MetricsGeocoderLogger, self).__init__(service_config.log_path)
-        self._service_config = service_config
+    def __init__(self, service_config, logger):
+        super(MetricsGeocoderLogger, self).__init__(service_config, logger)
 
-    def log(self, **data):
-        dump_data = self._dump_data(**data)
+    def log(self, data):
+        dump_data = self.collect_data(data)
         self.dump_to_file(dump_data)
 
-    def _dump_data(self, **data):
-        if data['success']:
-            cost = self._service_config.cost_per_hit
-            failed_rows = 0
-            successful_rows = 1
+    def collect_data(self, data):
+        dump_data = super(MetricsGeocoderLogger, self).collect_data(data)
+        if data.get('success', False):
+            cost = self.service_config.cost_per_hit
         else:
             cost = 0
-            failed_rows = 1
-            successful_rows = 0
 
-        if self._service_config.is_high_resolution:
+        if self.service_config.is_high_resolution:
             kind = 'high-resolution'
         else:
             kind = 'internal'
 
-        return {
+        dump_data.update({
             "batched": False,
             "cache_hits": 0,  # Always 0 because no cache involved
             # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211
             "cost": cost,
-            "created_at": datetime.now().isoformat(),
-            "failed_rows": failed_rows,
-            "geocoder_type": self._service_config.service_type,
+            "geocoder_type": self.service_config.service_type,
             "kind": kind,
-            "processable_rows": 1,
-            "processed_rows": successful_rows,
-            "real_rows": successful_rows,
-            "success": data['success'],
-            "successful_rows": successful_rows,
-            "username": self._service_config.username,
-            "organization": self._service_config.organization
-        }
+            "processed_rows": data.get('successful_rows', 0),
+            "real_rows": data.get('successful_rows', 0),
+        })
+
+        return dump_data
+
+
+class MetricsGenericLogger(MetricsLogger):
+
+    def __init__(self, service_config, logger):
+        super(MetricsGenericLogger, self).__init__(service_config, logger)
+
+    def log(self, data):
+        dump_data = self.collect_data(data)
+        self.dump_to_file(dump_data)
+
+    def collect_data(self, data):
+        return super(MetricsGenericLogger, self).collect_data(data)
+
+
+class MetricsIsolinesLogger(MetricsLogger):
+
+    def __init__(self, service_config, logger):
+        super(MetricsIsolinesLogger, self).__init__(service_config, logger)
+
+    def log(self, data):
+        dump_data = self.collect_data(data)
+        self.dump_to_file(dump_data)
+
+    def collect_data(self, data):
+        dump_data = super(MetricsIsolinesLogger, self).collect_data(data)
+        dump_data.update({
+            "isolines_generated": data.get('isolines_generated', 0)
+        })
+        return dump_data

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -1,5 +1,5 @@
 from user import UserMetricsService
-from log import MetricsLoggerFactory
+from log import MetricsDataGatherer
 from datetime import date
 import re
 
@@ -14,7 +14,6 @@ class QuotaService:
                                            redis_connection)
         self._user_service = UserMetricsService(self._user_service_config,
                                                 redis_connection)
-        self._metrics_logger = MetricsLoggerFactory.build(user_service_config)
 
     def check_user_quota(self):
         return self._quota_checker.check()
@@ -46,13 +45,19 @@ class QuotaService:
         self._user_service.increment_service_use(
             self._user_service_config.service_type, "isolines_generated",
             amount=amount)
+        MetricsDataGatherer.add('isolines_generated', amount)
 
     def _log_service_process(self, event):
-        if self._metrics_logger:
-            if event is 'success' or event is 'empty':
-                self._metrics_logger.log(success=True)
-            elif event is 'empty':
-                self._metrics_logger.log(success=False)
+        if event is 'success':
+            MetricsDataGatherer.add('success', True)
+            MetricsDataGatherer.add('successful_rows', 1)
+        elif event is 'empty':
+            MetricsDataGatherer.add('success', True)
+            MetricsDataGatherer.add('successful_rows', 1)
+            MetricsDataGatherer.add('empty_rows', 1)
+        elif event is 'fail':
+            MetricsDataGatherer.add('success', False)
+            MetricsDataGatherer.add('failed_rows', 1)
 
 
 class QuotaChecker:
@@ -93,7 +98,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_geocoding_limit = self._user_service_config.soft_geocoding_limit
 
-        if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -105,7 +110,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_isolines_limit = self._user_service_config.soft_isolines_limit
 
-        if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_isolines_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -115,8 +120,9 @@ class QuotaChecker:
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
+        soft_limit = self._user_service_config.soft_limit
 
-        if (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -128,7 +134,7 @@ class QuotaChecker:
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
 
-        if soft_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -8,6 +8,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
@@ -19,6 +20,8 @@ class UserMetricsService:
     def used_quota(self, service_type, date):
         if service_type == self.SERVICE_HERE_ISOLINES:
             return self.__used_isolines_quota(service_type, date)
+        elif service_type == self.SERVICE_MAPZEN_ROUTING:
+            return self.__used_routing_quota(service_type, date)
         else:
             return self.__used_geocoding_quota(service_type, date)
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.9.1',
+    version='0.10.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/__init__.py

@@ -0,0 +1,5 @@
+from test_helper import plpy_mock_config
+
+
+def setup():
+    plpy_mock_config()

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -0,0 +1,68 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from ..test_helper import *
+from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
+
+
+class TestRoutingConfig(TestCase):
+
+    def setUp(self):
+        self._redis_conn = MockRedis()
+        self._db_conn = plpy_mock
+        self._username = 'my_test_user'
+        self._user_key = "rails:users:{0}".format(self._username)
+        self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
+
+    def test_should_pick_quota_from_server_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1500000
+
+    def test_should_pick_quota_from_redis_if_present(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1000
+
+    def test_org_quota_overrides_user_quota(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = 'my_test_org'
+        orgname_key = "rails:orgs:{0}".format(orgname)
+        self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31')
+        self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000)
+
+        # TODO: these are not too relevant for the routing config
+        self._redis_conn.hset(orgname_key, 'geocoding_quota', 0)
+        self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
+
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 5000
+
+
+    def test_should_have_soft_limit_false_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == False
+
+    def test_can_set_soft_limit_in_user_conf(self):
+        self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == True
+
+
+class TestServicesRedisConfig(TestCase):
+    def test_it_picks_mapzen_routing_quota_from_redis(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', None)
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 42
+
+    def test_org_quota_overrides_user_quota(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme')
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 31415

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -0,0 +1,85 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from ..test_helper import *
+from cartodb_services.metrics.quota import QuotaChecker
+from cartodb_services.metrics import RoutingConfig
+from datetime import datetime
+
+
+class RoutingConfigMock(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestQuotaChecker(TestCase):
+
+    def setUp(self):
+        self.username = 'my_test_user'
+        self.period_end_date = datetime.today()
+        self.service_type = 'routing_mapzen'
+        self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format(
+            self.username,
+            self.service_type,
+            self.period_end_date.year,
+            self.period_end_date.month
+        )
+
+    def test_routing_quota_check_passes_when_enough_quota(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999)
+        assert QuotaChecker(user_service_config, redis_conn).check() == True
+
+    def test_routing_quota_check_fails_when_quota_exhausted(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_fails_right_in_the_limit(self):
+        """
+        I have 1000 credits and I just spent 1000 today. I should not pass
+        the check to perform the 1001th routing operation.
+        """
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = True
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == True

server/lib/python/cartodb_services/test/metrics/test_user.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+from cartodb_services.metrics import UserMetricsService
+import datetime
+from mockredis import MockRedis
+
+class UserGeocoderConfig(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestUserMetricsService(TestCase):
+
+    def setUp(self):
+        user_geocoder_config = UserGeocoderConfig(
+            username = 'my_test_user',
+            organization = None,
+            period_end_date = datetime.date.today()
+        )
+        redis_conn = MockRedis()
+        self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn)
+
+
+    def test_routing_used_quota_zero_when_no_usage(self):
+        assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0
+
+    def test_routing_used_quota_counts_usages(self):
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses')
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses')
+        assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2

server/lib/python/cartodb_services/test/mock_plpy.py

@@ -0,0 +1,53 @@
+import re
+
+
+class MockCursor:
+    def __init__(self, data):
+        self.cursor_pos = 0
+        self.data = data
+
+    def fetch(self, batch_size):
+        batch = self.data[self.cursor_pos: self.cursor_pos + batch_size]
+        self.cursor_pos += batch_size
+        return batch
+
+
+class MockPlPy:
+    def __init__(self):
+        self._reset()
+
+    def _reset(self):
+        self.infos = []
+        self.notices = []
+        self.debugs = []
+        self.logs = []
+        self.warnings = []
+        self.errors = []
+        self.fatals = []
+        self.executes = []
+        self.results = []
+        self.prepares = []
+        self.results = {}
+
+    def _define_result(self, query, result):
+        pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
+        self.results[pattern] = result
+
+    def notice(self, msg):
+        self.notices.append(msg)
+
+    def debug(self, msg):
+        self.notices.append(msg)
+
+    def info(self, msg):
+        self.infos.append(msg)
+
+    def cursor(self, query):
+        data = self.execute(query)
+        return MockCursor(data)
+
+    def execute(self, query, rows=1):
+        for pattern, result in self.results.iteritems():
+            if pattern.search(query):
+                return result
+        return []

server/lib/python/cartodb_services/test/test_config.py

@@ -1,20 +1,20 @@
-import test_helper
-from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
+from test_helper import *
 from unittest import TestCase
 from nose.tools import assert_raises
 from mockredis import MockRedis
 from datetime import datetime, timedelta
+from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
 
 
 class TestConfig(TestCase):
 
     def setUp(self):
         self.redis_conn = MockRedis()
-        self.plpy_mock = test_helper.build_plpy_mock()
+        plpy_mock_config()
 
     def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
-        test_helper.build_redis_user_config(self.redis_conn, 'test_user')
-        geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
+        build_redis_user_config(self.redis_conn, 'test_user')
+        geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          'test_user', None)
         assert geocoder_config.heremaps_geocoder is True
         assert geocoder_config.geocoding_quota == 100
@@ -22,10 +22,10 @@ class TestConfig(TestCase):
 
     def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
         yesterday = datetime.today() - timedelta(days=1)
-        test_helper.build_redis_user_config(self.redis_conn, 'test_user')
-        test_helper.build_redis_org_config(self.redis_conn, 'test_org',
-                                           quota=200, end_date=yesterday)
-        geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
+        build_redis_user_config(self.redis_conn, 'test_user')
+        build_redis_org_config(self.redis_conn, 'test_org',
+                               quota=200, end_date=yesterday)
+        geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          'test_user', 'test_org')
         assert geocoder_config.heremaps_geocoder is True
         assert geocoder_config.geocoding_quota == 200
@@ -34,10 +34,10 @@ class TestConfig(TestCase):
 
     def test_should_return_config_for_obs_snapshot(self):
         yesterday = datetime.today() - timedelta(days=1)
-        test_helper.build_redis_user_config(self.redis_conn, 'test_user',
-                                            do_quota=100, soft_do_limit=True,
-                                            end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
+        build_redis_user_config(self.redis_conn, 'test_user',
+                                do_quota=100, soft_do_limit=True,
+                                end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
                                           'test_user')
         assert do_config.monthly_quota == 100
         assert do_config.soft_limit is True
@@ -45,16 +45,16 @@ class TestConfig(TestCase):
 
     def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
         yesterday = datetime.today() - timedelta(days=1)
-        test_helper.build_redis_user_config(self.redis_conn, 'test_user',
-                                            end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
+        build_redis_user_config(self.redis_conn, 'test_user',
+                                end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
                                           'test_user')
         assert do_config.monthly_quota == 0
         assert do_config.soft_limit is False
         assert do_config.period_end_date.date() == yesterday.date()
 
     def test_should_raise_exception_when_missing_parameters(self):
-        plpy_mock = test_helper.build_plpy_mock(empty=True)
-        test_helper.build_redis_user_config(self.redis_conn, 'test_user')
+        plpy_mock._reset()
+        build_redis_user_config(self.redis_conn, 'test_user')
         assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
                       plpy_mock, 'test_user', None)

server/lib/python/cartodb_services/test/test_helper.py

@@ -1,7 +1,11 @@
 from datetime import datetime, date
-from mock import Mock
+from mock import Mock, MagicMock
+import random
 import sys
-sys.modules['plpy'] = Mock()
+from mock_plpy import MockPlPy
+
+plpy_mock = MockPlPy()
+sys.modules['plpy'] = plpy_mock
 
 
 def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None,
     redis_conn.zincrby(redis_name, date.day, amount)
 
 
-def build_plpy_mock(empty=False):
-    plpy_mock = Mock()
-    if not empty:
-        plpy_mock.execute.side_effect = _plpy_execute_side_effect
-
-    return plpy_mock
-
-
-def _plpy_execute_side_effect(*args, **kwargs):
-    if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
-        return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
-    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
-        return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
-    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
-        return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
-    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
-        return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
-    elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
-        return [{'conf': '{"environment": "testing"}'}]
+def plpy_mock_config():
+    plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
+    plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])

server/lib/python/cartodb_services/test/test_metrics_log.py

@@ -0,0 +1,23 @@
+from test_helper import *
+from cartodb_services.metrics import MetricsDataGatherer
+from unittest import TestCase
+from mock import Mock, MagicMock
+from nose.tools import assert_raises
+from datetime import datetime, date
+
+
+class TestMetricsDataGatherer(TestCase):
+
+    def setUp(self):
+        plpy_mock_config()
+
+
+    def test_should_use_multiple_instances_for_multiples_requests(self):
+        plpy_mock._define_result("select txid_current", [{'txid': 100}])
+        MetricsDataGatherer.add('test', 1)
+        plpy_mock._define_result("select txid_current", [{'txid': 101}])
+        MetricsDataGatherer.add('test', 2)
+        plpy_mock._define_result("select txid_current", [{'txid': 100}])
+        assert MetricsDataGatherer.get_element('test') is 1
+        plpy_mock._define_result("select txid_current", [{'txid': 101}])
+        assert MetricsDataGatherer.get_element('test') is 2

server/lib/python/cartodb_services/test/test_qps.py

@@ -1,4 +1,3 @@
-import test_helper
 import requests
 from unittest import TestCase
 from nose.tools import assert_raises

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -1,4 +1,4 @@
-import test_helper
+from test_helper import *
 from mockredis import MockRedis
 from cartodb_services.metrics import QuotaService
 from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
@@ -28,32 +28,32 @@ class TestQuotaService(TestCase):
 
     def test_should_return_true_if_user_quota_is_not_completely_used(self):
         qs = self.__build_geocoder_quota_service('test_user')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user')
+        increment_service_uses(self.redis_conn, 'test_user')
         assert qs.check_user_quota() is True
 
     def test_should_return_true_if_org_quota_is_not_completely_used(self):
         qs = self.__build_geocoder_quota_service('test_user',
                                                  orgname='test_org')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            orgname='test_org')
         assert qs.check_user_quota() is True
 
     def test_should_return_false_if_user_quota_is_surpassed(self):
         qs = self.__build_geocoder_quota_service('test_user')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            amount=300)
         assert qs.check_user_quota() is False
 
     def test_should_return_false_if_org_quota_is_surpassed(self):
         qs = self.__build_geocoder_quota_service('test_user',
                                                  orgname='test_org')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            orgname='test_org', amount=400)
         assert qs.check_user_quota() is False
 
     def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
         qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            amount=300)
         assert qs.check_user_quota() is True
 
@@ -61,7 +61,7 @@ class TestQuotaService(TestCase):
         qs = self.__build_geocoder_quota_service('test_user',
                                                  orgname='test_org',
                                                  soft_limit=True)
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                             orgname='test_org', amount=400)
         assert qs.check_user_quota() is True
 
@@ -140,18 +140,17 @@ class TestQuotaService(TestCase):
 
     def __prepare_quota_service(self, username, quota, service, orgname,
                                 soft_limit, do_quota, soft_do_limit, end_date):
-        test_helper.build_redis_user_config(self.redis_conn, username,
+        build_redis_user_config(self.redis_conn, username,
                                             quota=quota, service=service,
                                             soft_limit=soft_limit,
                                             soft_do_limit=soft_do_limit,
                                             do_quota=do_quota,
                                             end_date=end_date)
         if orgname:
-            test_helper.build_redis_org_config(self.redis_conn, orgname,
+            build_redis_org_config(self.redis_conn, orgname,
                                                quota=quota, service=service,
                                                do_quota=do_quota,
                                                end_date=end_date)
-        self._plpy_mock = test_helper.build_plpy_mock()
 
     def __build_geocoder_quota_service(self, username, quota=100,
                                        service='heremaps', orgname=None,
@@ -159,7 +158,7 @@ class TestQuotaService(TestCase):
                                        end_date=datetime.today()):
         self.__prepare_quota_service(username, quota, service, orgname,
                                      soft_limit, 0, False, end_date)
-        geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock,
+        geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          username, orgname)
         return QuotaService(geocoder_config, redis_connection=self.redis_conn)
 
@@ -168,7 +167,7 @@ class TestQuotaService(TestCase):
                                       quota=100, end_date=datetime.today()):
         self.__prepare_quota_service(username, quota, service, orgname,
                                      soft_limit, 0, False, end_date)
-        routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
+        routing_config = RoutingConfig(self.redis_conn, plpy_mock,
                                        username, orgname)
         return QuotaService(routing_config, redis_connection=self.redis_conn)
 
@@ -177,7 +176,7 @@ class TestQuotaService(TestCase):
                                       quota=100, end_date=datetime.today()):
         self.__prepare_quota_service(username, quota, service, orgname,
                                      soft_limit, 0, False, end_date)
-        isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock,
+        isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
                                                username, orgname)
         return QuotaService(isolines_config, redis_connection=self.redis_conn)
 
@@ -188,6 +187,6 @@ class TestQuotaService(TestCase):
                                                end_date=datetime.today()):
         self.__prepare_quota_service(username, 0, service, orgname, False,
                                      quota, soft_limit, end_date)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock,
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
                                           username, orgname)
         return QuotaService(do_config, redis_connection=self.redis_conn)

server/lib/python/cartodb_services/test/test_user_service.py

@@ -1,4 +1,4 @@
-import test_helper
+from test_helper import *
 from mockredis import MockRedis
 from cartodb_services.metrics import UserMetricsService
 from cartodb_services.metrics import GeocoderConfig
@@ -19,13 +19,13 @@ class TestUserService(TestCase):
 
     def test_user_used_quota_for_a_day(self):
         us = self.__build_user_service('test_user')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
 
     def test_org_used_quota_for_a_day(self):
         us = self.__build_user_service('test_user', orgname='test_org')
-        test_helper.increment_service_uses(self.redis_conn, 'test_user',
+        increment_service_uses(self.redis_conn, 'test_user',
                                            orgname='test_org',
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
@@ -135,14 +135,13 @@ class TestUserService(TestCase):
     def __build_user_service(self, username, quota=100, service='heremaps',
                              orgname=None, soft_limit=False,
                              end_date=date.today()):
-        test_helper.build_redis_user_config(self.redis_conn, username,
+        build_redis_user_config(self.redis_conn, username,
                                             quota=quota, service=service,
                                             soft_limit=soft_limit,
                                             end_date=end_date)
         if orgname:
-            test_helper.build_redis_org_config(self.redis_conn, orgname,
+            build_redis_org_config(self.redis_conn, orgname,
                                                quota=quota, end_date=end_date)
-        plpy_mock = test_helper.build_plpy_mock()
         geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          username, orgname)
         return UserMetricsService(geocoder_config, self.redis_conn)

test/helpers/import_helper.py

@@ -9,11 +9,11 @@ import time
 class ImportHelper:
 
     @classmethod
-    def import_test_dataset(cls, username, api_key, host):
+    def import_test_dataset(cls, username, api_key, host, schema):
         requests.packages.urllib3.disable_warnings()
-        url = "https://{0}.{1}/api/v1/imports/"\
-              "?type_guessing=false&privacy=public&api_key={2}".format(
-              username, host, api_key)
+        url = "{0}://{1}.{2}/api/v1/imports/"\
+              "?type_guessing=false&privacy=public&api_key={3}".format(
+              schema, username, host, api_key)
         dataset = {
             'file': open('fixtures/geocoder_api_test_dataset.csv', 'rb')}
         response = requests.post(url, files=dataset)
@@ -27,7 +27,8 @@ class ImportHelper:
                 username,
                 host,
                 api_key,
-                response_json['item_queue_id']
+                response_json['item_queue_id'],
+                schema
             )
             if table_name:
                 return table_name
@@ -35,10 +36,10 @@ class ImportHelper:
                 time.sleep(5)
 
     @classmethod
-    def get_imported_table_name(cls, username, host, api_key, import_id):
+    def get_imported_table_name(cls, username, host, api_key, import_id, schema):
         requests.packages.urllib3.disable_warnings()
-        import_url = "https://{0}.{1}/api/v1/imports/{2}?api_key={3}".format(
-            username, host, import_id, api_key)
+        import_url = "{0}://{1}.{2}/api/v1/imports/{3}?api_key={4}".format(
+            schema, username, host, import_id, api_key)
         import_data_response = requests.get(import_url)
         if import_data_response.status_code != 200:
             print "Error getting the table name from " \
@@ -49,10 +50,10 @@ class ImportHelper:
         return import_data_json['table_name']
 
     @classmethod
-    def clean_test_dataset(cls, username, api_key, table_name, host):
+    def clean_test_dataset(cls, username, api_key, table_name, host, schema):
         requests.packages.urllib3.disable_warnings()
-        url = "https://{0}.{1}/api/v2/sql?q=drop table {2}&api_key={3}".format(
-            username, host, table_name, api_key
+        url = "{0}://{1}.{2}/api/v2/sql?q=drop table {3}&api_key={4}".format(
+            schema, username, host, table_name, api_key
         )
         response = requests.get(url)
         if response.status_code != 200:

test/helpers/integration_test_helper.py

@@ -10,11 +10,13 @@ class IntegrationTestHelper:
         username = os.environ["GEOCODER_API_TEST_USERNAME"]
         api_key = os.environ["GEOCODER_API_TEST_API_KEY"]
         host = os.environ["GEOCODER_API_TEST_HOST"]
+        schema = os.environ["GEOCODER_API_TEST_SCHEMA"]
         table_name = os.environ["GEOCODER_API_TEST_TABLE_NAME"]
 
         return {
             "username": username,
             "api_key": api_key,
+            "schema": schema,
             "host": host,
             "table_name": table_name
         }

test/integration/test_admin0_functions.py

@@ -8,7 +8,8 @@ class TestAdmin0Functions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_admin1_functions.py

@@ -8,7 +8,8 @@ class TestAdmin1Functions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_data_observatory_functions.py

@@ -8,7 +8,8 @@ class TestDataObservatoryFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_ipaddress_functions.py

@@ -8,7 +8,8 @@ class TestPostalcodeFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_isolines_functions.py

@@ -8,7 +8,8 @@ class TestIsolinesFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_namedplace_functions.py

@@ -8,7 +8,8 @@ class TestNameplaceFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_postalcode_functions.py

@@ -8,7 +8,8 @@ class TestPostalcodeFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_routing_functions.py

@@ -8,7 +8,8 @@ class TestRoutingFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']

test/integration/test_street_functions.py

@@ -8,14 +8,15 @@ class TestStreetFunctions(TestCase):
 
     def setUp(self):
         self.env_variables = IntegrationTestHelper.get_environment_variables()
-        self.sql_api_url = "https://{0}.{1}/api/v1/sql".format(
+        self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
+            self.env_variables['schema'],
             self.env_variables['username'],
             self.env_variables['host'],
             self.env_variables['api_key']
         )
 
-    def test_if_select_with_street_point_is_ok(self):
-        query = "SELECT cdb_geocode_street_point(street) " \
+    def test_if_select_with_here_street_point_is_ok(self):
+        query = "SELECT cdb_here_geocode_street_point(street) " \
                 "as geometry FROM {0} LIMIT 1&api_key={1}".format(
                     self.env_variables['table_name'],
                     self.env_variables['api_key'])

test/run_tests.py

@@ -8,12 +8,13 @@ from helpers.import_helper import ImportHelper
 
 
 def main():
-    opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "host="])
+    opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "host=", "schema="])
 
     if len(args) < 2:
         usage()
         sys.exit()
 
+    schema = "https"
     host = "cartodb.com"
     username = args[0]
     api_key = args[1]
@@ -24,26 +25,31 @@ def main():
             sys.exit()
         elif o in ("--host"):
             host = opts[0][1]
+        elif o in ("--schema"):
+            schema = opts[1][1]
         else:
             assert False, "unhandled option"
 
     try:
-        table_name = ImportHelper.import_test_dataset(username, api_key, host)
-        set_environment_variables(username, api_key, table_name, host)
+        table_name = ImportHelper.import_test_dataset(username, api_key, host,
+                                                      schema)
+        set_environment_variables(username, api_key, table_name, host, schema)
         execute_tests()
     except Exception as e:
         print e.message
         sys.exit(1)
     finally:
         clean_environment_variables()
-        ImportHelper.clean_test_dataset(username, api_key, table_name, host)
+        ImportHelper.clean_test_dataset(username, api_key, table_name, host,
+                                        schema)
 
 
 def usage():
     print """Usage: run_tests.py [options] username api_key
         Options:
         -h: Show this help
-        --host: take that host as base (by default is cartodb.com)"""
+        --host: take that host as base (by default is cartodb.com)
+        --schema: define the url schema [http/https] (by default https)"""
 
 
 def execute_tests():
@@ -59,11 +65,12 @@ def execute_tests():
         sys.exit(1)
 
 
-def set_environment_variables(username, api_key, table_name, host):
+def set_environment_variables(username, api_key, table_name, host, schema):
     os.environ["GEOCODER_API_TEST_USERNAME"] = username
     os.environ["GEOCODER_API_TEST_API_KEY"] = api_key
     os.environ["GEOCODER_API_TEST_TABLE_NAME"] = table_name
     os.environ["GEOCODER_API_TEST_HOST"] = host
+    os.environ["GEOCODER_API_TEST_SCHEMA"] = schema
 
 
 def clean_environment_variables():
@@ -71,6 +78,7 @@ def clean_environment_variables():
     del os.environ["GEOCODER_API_TEST_API_KEY"]
     del os.environ["GEOCODER_API_TEST_TABLE_NAME"]
     del os.environ["GEOCODER_API_TEST_HOST"]
+    del os.environ["GEOCODER_API_TEST_SCHEMA"]
 
 if __name__ == "__main__":
     main()