From 5d5e8d6f9a3f1db9ba679832629ba0af75ca3ddb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 1 Aug 2016 14:00:59 +0200 Subject: [PATCH] Added logger with rollbar integration --- .../cartodb_services/mapzen/isolines.py | 13 ++--- .../cartodb_services/mapzen/matrix_client.py | 6 ++- .../cartodb_services/metrics/config.py | 13 +++++ .../cartodb_services/tools/__init__.py | 1 + .../cartodb_services/tools/log.py | 47 +++++++++++++++++++ .../python/cartodb_services/requirements.txt | 1 + server/lib/python/cartodb_services/setup.py | 2 +- 7 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/tools/log.py diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index de75459..daba485 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -1,5 +1,5 @@ from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2 -import logging + class MapzenIsolines: @@ -9,8 +9,9 @@ class MapzenIsolines: EARTH_RADIUS_METERS = 6367444 - def __init__(self, matrix_client): + def __init__(self, matrix_client, logger): self._matrix_client = matrix_client + self._logger = logger """Get an isochrone using mapzen API. @@ -85,11 +86,7 @@ class MapzenIsolines: def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0): # NOTE: not for production - #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log') - #logging.basicConfig(level=logging.DEBUG) - logging.debug('origin = %s' % origin) - logging.debug('costing_model = %s' % costing_model) - logging.debug('isorange = %d' % isorange) + self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange}) # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE @@ -114,7 +111,7 @@ class MapzenIsolines: else: costs[idx] = isorange - logging.debug('i = %d, costs = %s' % (i, costs)) + # self._logger.debug('i = %d, costs = %s' % (i, costs)) errors = [(cost - isorange) / float(isorange) for cost in costs] max_abs_error = max([abs(e) for e in errors]) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index bf72782..e73d8e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -18,8 +18,9 @@ class MatrixClient: ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many' - def __init__(self, matrix_key): + def __init__(self, matrix_key, logger): self._matrix_key = matrix_key + self._logger = logger """Get distances and times to a set of locations. See https://mapzen.com/documentation/matrix/api-reference/ @@ -40,6 +41,9 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params) + if not requests.codes.ok: + self._logger.warning('Error trying to get matrix distance from mapzen', data={"response": response, "locations": locations, "costing": costing}) + response.raise_for_status() # raise exception if not 200 OK return response.json() diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 05dd79e..70322af 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -16,6 +16,7 @@ class ServiceConfig(object): self._orgname = orgname self._db_config = ServicesDBConfig(db_conn, username, orgname) self._environment = self._db_config._server_environment + self._rollbar_api_key = self._db_config._rollbar_api_key if redis_connection: self._redis_config = ServicesRedisConfig(redis_connection).build( username, orgname) @@ -34,6 +35,10 @@ class ServiceConfig(object): def organization(self): return self._orgname + @property + def rollbar_api_key(self): + return self._rollbar_api_key + @property def environment(self): return self._environment @@ -468,6 +473,10 @@ class ServicesDBConfig: else: logger_conf = json.loads(logger_conf_json) self._geocoder_log_path = logger_conf['geocoder_log_path'] + if 'rollbar_api_key' in logger_conf: + self._rollbar_api_key = logger_conf['rollbar_api_key'] + else: + self._rollbar_api_key = None def _get_conf(self, key): try: @@ -529,6 +538,10 @@ class ServicesDBConfig: def geocoder_log_path(self): return self._geocoder_log_path + @property + def rollbar_api_key(self): + return self._rollbar_api_key + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index d884632..fdb533f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,3 +1,4 @@ from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine +from log import Logger diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py new file mode 100644 index 0000000..ae98e41 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -0,0 +1,47 @@ +import plpy +import rollbar +import sys +# Monkey path because plpython sys module doesn't have argv and rollbar +# package use it +sys.__dict__['argv'] = [] + + +class Logger: + + def __init__(self, config): + self._config = config + # We need to set the handler blocking (synchronous) because + # spawn a thread from plpython interpreter don't work + rollbar.init(self._config.rollbar_api_key, + self._config.environment, handler='blocking') + + def debug(self, text, exception=None, data={}): + self._send_to_rollbar('debug', text, exception, data) + plpy.debug(text) + + def info(self, text, exception=None, data={}): + self._send_to_rollbar('info', text, exception, data) + plpy.info(text) + + def warning(self, text, exception=None, data={}): + self._send_to_rollbar('warning', text, exception, data) + plpy.warning(text) + + def error(self, text, exception=None, data={}): + self._send_to_rollbar('error', text, exception, data) + plpy.error(text) + + def _send_to_rollbar(self, level, text, exception, data): + if self._rollbar_activated(): + try: + if exception: + rollbar.report_exc_info(exception, extra_data=data, + level=level) + else: + rollbar.report_message(text, level, extra_data=data) + except Exception as e: + plpy.warning('Error sending message/exception to rollbar: {0}'. + format(e)) + + def _rollbar_activated(self): + return True if self._config.rollbar_api_key else False diff --git a/server/lib/python/cartodb_services/requirements.txt b/server/lib/python/cartodb_services/requirements.txt index 61426a1..1b8b873 100644 --- a/server/lib/python/cartodb_services/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -2,6 +2,7 @@ redis==2.10.5 hiredis==0.1.5 python-dateutil==2.2 googlemaps==2.4.2 +rollbar==0.13.2 # Dependency for googlemaps package requests<=2.9.1 diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index f87881f..71faaf8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.7.2.3', + version='0.7.4', description='CartoDB Services API Python Library',