Merge pull request #110 from CartoDB/qps_for_mapzen_services

QPS manager for Mapzen

server/lib/python/cartodb_services/cartodb_services/mapzen/exceptions.py

@@ -14,3 +14,8 @@ class WrongParams(Exception):
 class MalformedResult(Exception):
     def __str__(self):
             return repr('Result structure is malformed')
+
+
+class TimeoutException(Exception):
+    def __str__(self):
+            return repr('Timeout requesting to mapzen server')

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -0,0 +1,47 @@
+import time
+from datetime import datetime
+from exceptions import TimeoutException
+
+DEFAULT_RETRY_TIMEOUT = 60
+
+
+def qps_retry(f):
+    def wrapped_f(*args, **kw):
+        return QPSService().call(f, *args, **kw)
+    return wrapped_f
+
+
+class QPSService:
+
+    def __init__(self, queries_per_second=10,
+                 retry_timeout=DEFAULT_RETRY_TIMEOUT):
+        self._queries_per_second = queries_per_second
+        self._retry_timeout = retry_timeout
+
+    def call(self, fn, *args, **kwargs):
+        start_time = datetime.now()
+        attempt_number = 1
+        while True:
+            try:
+                return fn(*args, **kwargs)
+            except Exception as e:
+                if hasattr(e, 'response') and e.response.status_code == 429:
+                    self.retry(start_time, attempt_number)
+                else:
+                    raise e
+            attempt_number += 1
+
+    def retry(self, first_request_time, retry_count):
+        elapsed = datetime.now() - first_request_time
+        if elapsed.seconds > self._retry_timeout:
+            raise TimeoutException()
+
+        # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
+        # iteration.
+        delay = (1.0/self._queries_per_second) * 1.5 ** retry_count
+
+        # https://www.awsarchitectureblog.com/2015/03/backoff.html
+        # https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py#L193
+        sleep_time = delay * (random.random() + 0.5)
+
+        time.sleep(sleep_time)

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -2,7 +2,8 @@ import requests
 import json
 import re
 
-from exceptions import WrongParams
+from exceptions import WrongParams, MalformedResult
+from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
 
 
@@ -31,6 +32,7 @@ class MapzenRouting:
         self._app_key = app_key
         self._url = base_url
 
+    @qps_retry
     def calculate_route_point_to_point(self, origin, destination, mode,
                                        options=[], units=METRICS_UNITS):
         parsed_options = self.__parse_options(options)

server/lib/python/cartodb_services/test/test_mapzenrouting.py

@@ -60,18 +60,24 @@ class MapzenRoutingTestCase(unittest.TestCase):
         self.url = MapzenRouting.PRODUCTION_ROUTING_BASE_URL
 
     def test_calculate_simple_routing_with_valid_params(self, req_mock):
-        req_mock.register_uri('GET', requests_mock.ANY, text=self.GOOD_RESPONSE)
-        origin = Coordinate('-120.2','38.5')
-        destination = Coordinate('-126.4','43.2')
-        response = self.routing.calculate_route_point_to_point(origin, destination,'car')
+        req_mock.register_uri('GET', requests_mock.ANY,
+                              text=self.GOOD_RESPONSE)
+        origin = Coordinate('-120.2', '38.5')
+        destination = Coordinate('-126.4', '43.2')
+        response = self.routing.calculate_route_point_to_point(origin,
+                                                               destination,
+                                                               'car')
 
         self.assertEqual(response.shape, self.GOOD_SHAPE)
         self.assertEqual(response.length, 444.59)
         self.assertEqual(response.duration, 16969)
 
     def test_uknown_mode_raise_exception(self, req_mock):
-        req_mock.register_uri('GET', requests_mock.ANY, text=self.GOOD_RESPONSE)
-        origin = Coordinate('-120.2','38.5')
-        destination = Coordinate('-126.4','43.2')
+        req_mock.register_uri('GET', requests_mock.ANY,
+                              text=self.GOOD_RESPONSE)
+        origin = Coordinate('-120.2', '38.5')
+        destination = Coordinate('-126.4', '43.2')
 
-        assert_raises(WrongParams, self.routing.calculate_route_point_to_point, origin, destination, 'unknown')
+        assert_raises(WrongParams,
+                      self.routing.calculate_route_point_to_point,
+                      origin, destination, 'unknown')