Merge pull request #315 from CartoDB/add_retries_to_requests

Make the HTTP requests retry on timeout

NEWS.md

@@ -1,3 +1,28 @@
+November 29st, 2016
+===================
+* Version 0.20.0 of the server and version 0.12.0 of the python library
+  * Added integration with the new Mapzen isochrones functionality
+
+November 25st, 2016
+===================
+* Version 0.19.0 of the server, version 0.11.0 of the python library and version 0.13.0 of the client
+  * functions to check the quota, both server and client
+  * removed the no_params from the templates (this caused trouble, not needed anymore)
+  * bug fixes: observatory quota, quotas as integers, mapzen geocoder soft limit as bool
+
+November 21st, 2016
+===================
+* Version 0.18.1 of the server and version 0.12.1 of the client
+  * Add new fields to the obs_meta_geometry due to new changes introduced in the DO 1.1.2 release
+
+November 11st, 2016
+===================
+* Version 0.18.0 of the server and version 0.12.0 of the client
+  * Added obs_legacybuildermetada functions to grab the needed metadata in the builder while making a data enrichment analysis. Closes #286
+  * Added metadata functions that will be used in the future to gather the metadata for the new data enrichment UI. Closes #287
+  * Fixed integration test for street geocoding
+  * Makefile now has a new task to create a new release in the client part, as was in the server, using make release NEW_VERSION=x.x.x
+
 November 7st, 2016
 ==================
 * Version 0.17.0 of the server and version 0.10.0 of the python package

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -4,6 +4,7 @@
 import json
 import requests
 
+from requests.adapters import HTTPAdapter
 from exceptions import *
 from cartodb_services.metrics import Traceable
 
@@ -17,6 +18,7 @@ class HereMapsGeocoder(Traceable):
     DEFAULT_GEN = 9
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=3
 
     ADDRESS_PARAMS = [
         'city',
@@ -88,7 +90,10 @@ class HereMapsGeocoder(Traceable):
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self.host, HTTPAdapter(self.MAX_RETRIES))
+        response = session.get(self.host, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 
 from exceptions import WrongParams
+from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 
 
@@ -13,6 +14,7 @@ class HereMapsRoutingIsoline(Traceable):
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 3
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -53,6 +55,9 @@ class HereMapsRoutingIsoline(Traceable):
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
         response = requests.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenGeocoder(Traceable):
     BASE_URL = 'https://search.mapzen.com/v1/search'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 3
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -27,7 +29,10 @@ class MapzenGeocoder(Traceable):
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
             self.add_response_data(response, self._logger)
             if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 
@@ -12,6 +13,7 @@ class MapzenIsochrones:
     BASE_URL = 'https://matrix.mapzen.com/isochrone'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 3
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -28,7 +30,10 @@ class MapzenIsochrones:
         request_params = self._parse_request_params(locations, costing,
                                                     ranges)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT,
                                             self.READ_TIMEOUT))
 

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenRouting(Traceable):
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=3
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -46,7 +48,10 @@ class MapzenRouting(Traceable):
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
+        response = session.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.12.0',
+    version='0.12.1',
 
     description='CartoDB Services API Python Library',