Added CR suggestions

2018-01-04 12:30:58 +01:00
parent cc34a8b19b
commit 51d97228dc
18 changed files with 103 additions and 14732 deletions
@@ -4,7 +4,7 @@ import re

 from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
-from qps import qps_retry
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
 from cartodb_services.metrics import Traceable

@@ -4,7 +4,7 @@ import re

 from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
-from qps import qps_retry
+from cartodb_services.tools.qps import qps_retry


 class MapzenIsochrones:
@@ -1,6 +1,6 @@
 import requests
 import json
-from qps import qps_retry
+from cartodb_services.tools.qps import qps_retry
 from exceptions import ServiceException
 from cartodb_services.metrics import Traceable

@@ -1,69 +0,0 @@
-import time
-import random
-from datetime import datetime
-from exceptions import TimeoutException
-
-DEFAULT_RETRY_TIMEOUT = 60
-DEFAULT_QUERIES_PER_SECOND = 10
-
-def qps_retry(original_function=None,**options):
-    """ Query Per Second retry decorator
-        The intention of this decorator is to retry requests against third
-        party services that has QPS restriction.
-        Parameters:
-            - timeout: Maximum number of seconds to retry
-            - qps: Allowed queries per second. This parameter is used to
-                   calculate the next time to retry the request
-    """
-    if original_function is not None:
-        def wrapped_function(*args, **kwargs):
-            if 'timeout' in options:
-                timeout = options['timeout']
-            else:
-                timeout = DEFAULT_RETRY_TIMEOUT
-            if 'qps' in options:
-                qps = options['qps']
-            else:
-                qps = DEFAULT_QUERIES_PER_SECOND
-            return QPSService(retry_timeout=timeout, queries_per_second=qps).call(original_function, *args, **kwargs)
-        return wrapped_function
-    else:
-        def partial_wrapper(func):
-            return qps_retry(func, **options)
-        return partial_wrapper
-
-
-class QPSService:
-
-    def __init__(self, queries_per_second, retry_timeout):
-        self._queries_per_second = queries_per_second
-        self._retry_timeout = retry_timeout
-
-    def call(self, fn, *args, **kwargs):
-        start_time = datetime.now()
-        attempt_number = 1
-        while True:
-            try:
-                return fn(*args, **kwargs)
-            except Exception as e:
-                response = getattr(e, 'response', None)
-                if response is not None and (response.status_code == 429):
-                    self.retry(start_time, attempt_number)
-                else:
-                    raise e
-            attempt_number += 1
-
-    def retry(self, first_request_time, retry_count):
-        elapsed = datetime.now() - first_request_time
-        if elapsed.total_seconds()  > self._retry_timeout:
-            raise TimeoutException()
-
-        # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
-        # iteration.
-        delay = (1.0/self._queries_per_second) * 1.5 ** retry_count
-
-        # https://www.awsarchitectureblog.com/2015/03/backoff.html
-        # https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py#L193
-        sleep_time = delay * (random.random() + 0.5)
-
-        time.sleep(sleep_time)
@@ -4,7 +4,7 @@ import re

 from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
-from qps import qps_retry
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
 from cartodb_services.metrics import MetricsDataGatherer, Traceable

@@ -35,18 +35,3 @@ def coordinates_to_polygon(coordinates):
        geometry = None

    return geometry
-
-
-def country_to_iso3(country):
-    """ Convert country to its iso3 code """
-    try:
-        country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
-                                    "'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
-        country_result = plpy.execute(country_plan, [country], 1)
-        if country_result:
-            return country_result[0]['iso3']
-        else:
-            return None
-    except BaseException as e:
-        plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
-        return None
@@ -599,19 +599,19 @@ class ServicesDBConfig:
    def _get_mapbox_config(self):
        mapbox_conf_json = self._get_conf('mapbox_conf')
        if not mapbox_conf_json:
-            raise ConfigException('Mapzen configuration missing')
+            raise ConfigException('Mapbox configuration missing')
        else:
-            mapzen_conf = json.loads(mapbox_conf_json)
-            self._mapbox_matrix_api_key = mapzen_conf['matrix']['api_key']
-            self._mapbox_matrix_quota = mapzen_conf['matrix']['monthly_quota']
-            self._mapbox_matrix_service_params = mapzen_conf['matrix'].get('service', {})
-            self._mapbox_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
-            self._mapbox_routing_api_key = mapzen_conf['routing']['api_key']
-            self._mapbox_routing_quota = mapzen_conf['routing']['monthly_quota']
-            self._mapbox_routing_service_params = mapzen_conf['routing'].get('service', {})
-            self._mapbox_geocoder_api_key = mapzen_conf['geocoder']['api_key']
-            self._mapbox_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
-            self._mapbox_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
+            mapbox_conf = json.loads(mapbox_conf_json)
+            self._mapbox_matrix_api_key = mapbox_conf['matrix']['api_key']
+            self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
+            self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
+            self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
+            self._mapbox_routing_api_key = mapbox_conf['routing']['api_key']
+            self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
+            self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
+            self._mapbox_geocoder_api_key = mapbox_conf['geocoder']['api_key']
+            self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
+            self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})

    def _get_data_observatory_config(self):
        do_conf_json = self._get_conf('data_observatory_conf')
@@ -72,6 +72,9 @@ class QuotaChecker:
        if re.match('geocoder_*',
                    self._user_service_config.service_type) is not None:
            return self.__check_geocoder_quota()
+        elif re.match('here_isolines',
+                      self._user_service_config.service_type) is not None:
+            return self.__check_isolines_quota()
        elif re.match('mapzen_isolines',
                      self._user_service_config.service_type) is not None:
            return self.__check_isolines_quota()
@@ -1,3 +1,3 @@
 class TimeoutException(Exception):
    def __str__(self):
-        return repr('Timeout requesting to mapzen server')
+        return repr('Timeout requesting to server')