New metrics logger system

- It's not mandatory to have the log_path defined for all the services
- Now we add metrics to a MetricsGatherer using the add method in order
  to gather data from all the function scope, not just in one defined
  point
- There is a `metrics` context manager to wrap the function block. This
  context manager get the execution time for the block among other
  things

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -2,14 +2,100 @@ from datetime import datetime
 import abc
 import json
 import re
+import time
+from contextlib import contextmanager
 
 
-class MetricsLoggerFactory:
+@contextmanager
+def metrics(function, service_config):
+    try:
+        start_time = time.time()
+        yield
+    finally:
+        end_time = time.time()
+        MetricsDataGatherer.add('function_name', function)
+        MetricsDataGatherer.add('function_execution_time', (end_time - start_time))
+        logger = MetricsServiceLoggerFactory.build(service_config)
+        if logger:
+            data = MetricsDataGatherer.get()
+            logger.log(data)
+        MetricsDataGatherer.clean()
+
+
+class Traceable:
+
+    def add_response_data(self, response):
+        try:
+            response_data = {}
+            response_data['time'] = response.elapsed.total_seconds()
+            response_data['code'] = response.status_code
+            response_data['message'] = response.reason
+            stored_data = MetricsDataGatherer.get_element('response')
+            if stored_data:
+                stored_data.append(response_data)
+            else:
+                MetricsDataGatherer.add('response', [response_data])
+        except BaseException as e:
+            # We don't want to stop the job for some error here
+            plpy.warning(e)
+
+
+class MetricsDataGatherer:
+
+    class __MetricsDataGatherer:
+        def __init__(self):
+            self.data = {}
+
+        def add(self, key, value):
+            self.data[key] = value
+
+        def get(self):
+            return self.data
+
+        def get_element(self, key):
+            return self.data.get(key, None)
+
+        def clean(self):
+            self.data = {}
+
+    __instance = None
+
+    @classmethod
+    def add(self, key, value):
+        MetricsDataGatherer.instance().add(key, value)
+
+    @classmethod
+    def get(self):
+        return MetricsDataGatherer.instance().get()
+
+    @classmethod
+    def get_element(self, key):
+        return MetricsDataGatherer.instance().get_element(key)
+
+    @classmethod
+    def clean(self):
+        MetricsDataGatherer.instance().clean()
+
+    @classmethod
+    def instance(self):
+        if not MetricsDataGatherer.__instance:
+            MetricsDataGatherer.__instance = MetricsDataGatherer.__MetricsDataGatherer()
+
+        return MetricsDataGatherer.__instance
+
+
+class MetricsServiceLoggerFactory:
 
     @classmethod
     def build(self, service_config):
-        if re.match('geocoder_*', service_config.service_type):
+        if re.search('^geocoder_*', service_config.service_type):
             return MetricsGeocoderLogger(service_config)
+        elif re.search('^routing_*', service_config.service_type):
+            return MetricsGenericLogger(service_config)
+        elif re.search('_isolines$', service_config.service_type):
+            return MetricsIsolinesLogger(service_config)
+        elif re.search('^obs_*', service_config.service_type):
+            return MetricsGenericLogger(service_config)
         else:
             return None
 
@@ -17,58 +103,103 @@ class MetricsLoggerFactory:
 class MetricsLogger(object):
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, file_path):
-        self._file_path = file_path
+    def __init__(self, service_config):
+        self._service_config = service_config
 
     def dump_to_file(self, data):
-        with open(self._file_path, 'a') as logfile:
-            json.dump(data, logfile)
-            logfile.write('\n')
+        log_path = self.service_config.metrics_log_path
+        if log_path:
+            with open(log_path, 'a') as logfile:
+                json.dump(data, logfile)
+                logfile.write('\n')
+
+    def collect_data(self, data):
+        return {
+            "function_name": data.get('function_name', None),
+            "function_execution_time": data.get('function_execution_time', None),
+            "service": self._service_config.service_type,
+            "processable_rows": 1,
+            "success": data.get('success', False),
+            "successful_rows": data.get('successful_rows', 0),
+            "failed_rows": data.get('failed_rows', 0),
+            "empty_rows": data.get('empty_rows', 0),
+            "created_at": datetime.now().isoformat(),
+            "provider": self._service_config.provider,
+            "username": self._service_config.username,
+            "organization": self._service_config.organization,
+            "response": data.get('response', None)
+        }
+
+    @property
+    def service_config(self):
+        return self._service_config
 
     @abc.abstractproperty
-    def log(self, **data):
+    def log(self, data):
         raise NotImplementedError('log method must be defined')
 
 
 class MetricsGeocoderLogger(MetricsLogger):
 
     def __init__(self, service_config):
-        super(MetricsGeocoderLogger, self).__init__(service_config.log_path)
-        self._service_config = service_config
+        super(MetricsGeocoderLogger, self).__init__(service_config)
 
-    def log(self, **data):
-        dump_data = self._dump_data(**data)
+    def log(self, data):
+        dump_data = self.collect_data(data)
         self.dump_to_file(dump_data)
 
-    def _dump_data(self, **data):
-        if data['success']:
-            cost = self._service_config.cost_per_hit
-            failed_rows = 0
-            successful_rows = 1
+    def collect_data(self, data):
+        dump_data = super(MetricsGeocoderLogger, self).collect_data(data)
+        if data.get('success', False):
+            cost = self.service_config.cost_per_hit
         else:
             cost = 0
-            failed_rows = 1
-            successful_rows = 0
 
-        if self._service_config.is_high_resolution:
+        if self.service_config.is_high_resolution:
             kind = 'high-resolution'
         else:
             kind = 'internal'
 
-        return {
+        dump_data.update({
             "batched": False,
             "cache_hits": 0,  # Always 0 because no cache involved
             # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211
             "cost": cost,
-            "created_at": datetime.now().isoformat(),
-            "failed_rows": failed_rows,
-            "geocoder_type": self._service_config.service_type,
+            "geocoder_type": self.service_config.service_type,
             "kind": kind,
-            "processable_rows": 1,
-            "processed_rows": successful_rows,
-            "real_rows": successful_rows,
-            "success": data['success'],
-            "successful_rows": successful_rows,
-            "username": self._service_config.username,
-            "organization": self._service_config.organization
-        }
+            "processed_rows": data.get('successful_rows', 0),
+            "real_rows": data.get('successful_rows', 0),
+        })
+
+        return dump_data
+
+
+class MetricsGenericLogger(MetricsLogger):
+
+    def __init__(self, service_config):
+        super(MetricsGenericLogger, self).__init__(service_config)
+
+    def log(self, data):
+        dump_data = self.collect_data(data)
+        self.dump_to_file(dump_data)
+
+    def collect_data(self, data):
+        return super(MetricsGenericLogger, self).collect_data(data)
+
+class MetricsIsolinesLogger(MetricsLogger):
+
+    def __init__(self, service_config):
+        super(MetricsIsolinesLogger, self).__init__(service_config)
+
+    def log(self, data):
+        dump_data = self.collect_data(data)
+        self.dump_to_file(dump_data)
+
+    def collect_data(self, data):
+        dump_data =  super(MetricsIsolinesLogger, self).collect_data(data)
+
+        dump_data.update({
+            "isolines_generated": data.get('isolines_generated', 0)
+        })
+
+        return dump_data