Merge pull request #298 from CartoDB/development

Release Python 0.9.4

NEWS.md

@@ -1,3 +1,8 @@
+October 21st, 2016
+==================
+* Version 0.9.2 of the python package
+    * mapzen routing quota now is configurable per user
+
 September 28, 2016
 ==========
 * Released version 0.8.1 of Python package cartodb\_services

doc/demographic_functions.md

@@ -1,6 +1,6 @@
 # Demographic Functions
 
-The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/carto-engine/dataservices-api/geocoding-functions/) documentation.
+The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
 
 _**Note:** The Demographic Snapshot functions are only available for the United States._
 

doc/geocoding_functions.md

@@ -290,8 +290,7 @@ Geocodes a postal code from a specified country into an IP address, displayed as
 
 Name | Type | Description
 --- | --- | ---
-`ip_address` | `text` | Postal code
-`country_name` | `text` | IPv4 or IPv6 address
+`ip_address` | `text` | IPv4 or IPv6 address
 
 #### Returns
 

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -14,6 +14,8 @@ class HereMapsGeocoder:
     STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
     DEFAULT_MAXRESULTS = 1
     DEFAULT_GEN = 9
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ADDRESS_PARAMS = [
         'city',
@@ -85,7 +87,8 @@ class HereMapsGeocoder:
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params)
+        response = requests.get(self.host, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         if response.status_code == requests.codes.ok:
             return json.loads(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -10,6 +10,8 @@ class HereMapsRoutingIsoline:
     PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
     STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com'
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -50,7 +52,8 @@ class HereMapsRoutingIsoline:
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
-        response = requests.get(self._url, params=request_params)
+        response = requests.get(self._url, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         if response.status_code == requests.codes.ok:
             return self.__parse_isolines_response(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -11,6 +11,8 @@ class MapzenGeocoder:
     'A Mapzen Geocoder wrapper for python'
 
     BASE_URL = 'https://search.mapzen.com/v1/search'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -24,7 +26,8 @@ class MapzenGeocoder:
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params)
+            response = requests.get(self._url, params=request_params,
+                                    timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
             if response.status_code == requests.codes.ok:
                 return self.__parse_response(response.text)
             elif response.status_code == requests.codes.bad_request:
@@ -41,6 +44,12 @@ class MapzenGeocoder:
                                       "state_province": state_province})
                 raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
                                        response)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            self._logger.error('Timeout connecting to Mapzen geocoding server')
+            raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
+                                    None)
         except requests.ConnectionError as e:
             # Don't raise the exception to continue with the geocoding job
             self._logger.error('Error connecting to Mapzen geocoding server',

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -86,7 +86,7 @@ class MapzenIsolines:
     def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
 
         # NOTE: not for production
-        self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
+        # self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
 
         # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
         # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
@@ -105,14 +105,16 @@ class MapzenIsolines:
 
             response = self._matrix_client.one_to_many([origin] + location_estimates,  costing_model)
             costs = [None] * self.NUMBER_OF_ANGLES
+            if not response:
+                # In case the matrix client doesn't return any data
+                break
+
             for idx, c in enumerate(response['one_to_many'][0][1:]):
                 if c[cost_variable]:
                     costs[idx] = c[cost_variable]*unit_factor
                 else:
                     costs[idx] = isorange
 
-            # self._logger.debug('i = %d, costs = %s' % (i, costs))
-
             errors = [(cost - isorange) / float(isorange) for cost in costs]
             max_abs_error = max([abs(e) for e in errors])
             if max_abs_error <= self.TOLERANCE:

server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py

@@ -19,6 +19,8 @@ class MatrixClient:
     """
 
     ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     def __init__(self, matrix_key, logger):
         self._matrix_key = matrix_key
@@ -41,9 +43,10 @@ class MatrixClient:
             'costing': costing,
             'api_key': self._matrix_key
         }
-        response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
+        response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
 
-        if not requests.codes.ok:
+        if response.status_code != requests.codes.ok:
             self._logger.error('Error trying to get matrix distance from mapzen',
                                data={"response_status": response.status_code,
                                      "response_reason": response.reason,
@@ -52,6 +55,22 @@ class MatrixClient:
                                      "response_headers": response.headers,
                                      "locations": locations,
                                      "costing": costing})
-            raise ServiceException("Error trying to get matrix distance from mapzen", response)
+            # In case 4xx error we return empty because the error comes from
+            # the provided info by the user and we don't want to top the
+            # isolines generation
+            if response.status_code == requests.codes.bad_request:
+                return {}
+            elif response.status_code == 504:
+                # Due to some unsolved problems in the Mapzen Matrix API we're
+                # getting randomly 504, probably timeouts. To avoid raise an
+                # exception in all the jobs, for now we're going to return
+                # empty in that case
+                return {}
+            else:
+                raise ServiceException("Error trying to get matrix distance from mapzen", response)
 
-        return response.json()
+        # response could return with empty json
+        try:
+            return response.json()
+        except:
+            return {}

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -11,6 +11,8 @@ class MapzenRouting:
     'A Mapzen Routing wrapper for python'
 
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -43,7 +45,8 @@ class MapzenRouting:
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params)
+        response = requests.get(self._url, params=request_params,
+                                timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         if response.status_code == requests.codes.ok:
             return self.__parse_routing_response(response.text)
         elif response.status_code == requests.codes.bad_request:

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -116,6 +116,8 @@ class RoutingConfig(ServiceConfig):
     ROUTING_PROVIDER_KEY = 'routing_provider'
     MAPZEN_PROVIDER = 'mapzen'
     DEFAULT_PROVIDER = 'mapzen'
+    QUOTA_KEY = 'mapzen_routing_quota'
+    SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
         super(RoutingConfig, self).__init__(redis_connection, db_conn,
@@ -124,7 +126,8 @@ class RoutingConfig(ServiceConfig):
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
         self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
+        self._set_monthly_quota()
+        self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
     @property
@@ -144,6 +147,28 @@ class RoutingConfig(ServiceConfig):
     def period_end_date(self):
         return self._period_end_date
 
+    @property
+    def soft_limit(self):
+        return self._soft_limit
+
+
+    def _set_monthly_quota(self):
+        self._monthly_quota = self._get_effective_monthly_quota()
+
+    def _get_effective_monthly_quota(self):
+        quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
+        if quota_from_redis and quota_from_redis <> '':
+            return int(quota_from_redis)
+        else:
+            return self._db_config.mapzen_routing_monthly_quota
+
+    def _set_soft_limit(self):
+        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
+            self._soft_limit = True
+        else:
+            self._soft_limit = False
+
+
 
 class IsolinesRoutingConfig(ServiceConfig):
 
@@ -547,6 +572,7 @@ class ServicesRedisConfig:
     GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
+    ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
     OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
     OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
@@ -585,8 +611,12 @@ class ServicesRedisConfig:
         if not org_config:
             raise ConfigException("""There is no organization config available. Please check your configuration.'""")
         else:
-            user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
-            user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+            if self.QUOTA_KEY in org_config:
+                user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
+            if self.ISOLINES_QUOTA_KEY in org_config:
+                user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
+            if self.ROUTING_QUOTA_KEY in org_config:
+                user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
             if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
                 user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
             if self.OBS_GENERAL_QUOTA_KEY in org_config:

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -93,7 +93,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_geocoding_limit = self._user_service_config.soft_geocoding_limit
 
-        if soft_geocoding_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_geocoding_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -105,7 +105,7 @@ class QuotaChecker:
         current_used = self._user_service.used_quota(service_type, today)
         soft_isolines_limit = self._user_service_config.soft_isolines_limit
 
-        if soft_isolines_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_isolines_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -115,8 +115,9 @@ class QuotaChecker:
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
+        soft_limit = self._user_service_config.soft_limit
 
-        if (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False
@@ -128,7 +129,7 @@ class QuotaChecker:
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)
 
-        if soft_limit or (user_quota > 0 and current_used <= user_quota):
+        if soft_limit or (user_quota > 0 and current_used < user_quota):
             return True
         else:
             return False

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -8,6 +8,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
@@ -19,6 +20,8 @@ class UserMetricsService:
     def used_quota(self, service_type, date):
         if service_type == self.SERVICE_HERE_ISOLINES:
             return self.__used_isolines_quota(service_type, date)
+        elif service_type == self.SERVICE_MAPZEN_ROUTING:
+            return self.__used_routing_quota(service_type, date)
         else:
             return self.__used_geocoding_quota(service_type, date)
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.9.1',
+    version='0.9.4',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
+from ..test_helper import build_plpy_mock
+
+class TestRoutingConfig(TestCase):
+
+    def setUp(self):
+        self._redis_conn = MockRedis()
+        self._db_conn = build_plpy_mock()
+        self._username = 'my_test_user'
+        self._user_key = "rails:users:{0}".format(self._username)
+        self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
+
+    def test_should_pick_quota_from_server_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1500000
+
+    def test_should_pick_quota_from_redis_if_present(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 1000
+
+    def test_org_quota_overrides_user_quota(self):
+        self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
+        orgname = 'my_test_org'
+        orgname_key = "rails:orgs:{0}".format(orgname)
+        self._redis_conn.hset(orgname_key, 'period_end_date', '2016-05-31')
+        self._redis_conn.hset(orgname_key, 'mapzen_routing_quota', 5000)
+
+        # TODO: these are not too relevant for the routing config
+        self._redis_conn.hset(orgname_key, 'geocoding_quota', 0)
+        self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
+
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.monthly_quota == 5000
+
+
+    def test_should_have_soft_limit_false_by_default(self):
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == False
+
+    def test_can_set_soft_limit_in_user_conf(self):
+        self._redis_conn.hset(self._user_key, 'soft_mapzen_routing_limit', True)
+        orgname = None
+        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
+        assert config.soft_limit == True
+
+
+class TestServicesRedisConfig(TestCase):
+    def test_it_picks_mapzen_routing_quota_from_redis(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', None)
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 42
+
+    def test_org_quota_overrides_user_quota(self):
+        redis_conn = MockRedis()
+        redis_conn.hset('rails:users:my_username', 'mapzen_routing_quota', 42)
+        redis_conn.hset('rails:orgs:acme', 'mapzen_routing_quota', 31415)
+        redis_config = ServicesRedisConfig(redis_conn).build('my_username', 'acme')
+        assert 'mapzen_routing_quota' in redis_config
+        assert int(redis_config['mapzen_routing_quota']) == 31415

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -0,0 +1,85 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from ..test_helper import build_plpy_mock
+from cartodb_services.metrics.quota import QuotaChecker
+from cartodb_services.metrics import RoutingConfig
+from datetime import datetime
+
+
+class RoutingConfigMock(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestQuotaChecker(TestCase):
+
+    def setUp(self):
+        self.username = 'my_test_user'
+        self.period_end_date = datetime.today()
+        self.service_type = 'routing_mapzen'
+        self.redis_key = 'user:{0}:{1}:success_responses:{2}{3}'.format(
+            self.username,
+            self.service_type,
+            self.period_end_date.year,
+            self.period_end_date.month
+        )
+
+    def test_routing_quota_check_passes_when_enough_quota(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 999)
+        assert QuotaChecker(user_service_config, redis_conn).check() == True
+
+    def test_routing_quota_check_fails_when_quota_exhausted(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_fails_right_in_the_limit(self):
+        """
+        I have 1000 credits and I just spent 1000 today. I should not pass
+        the check to perform the 1001th routing operation.
+        """
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = False
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1000)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == False
+
+    def test_routing_quota_check_passes_if_no_quota_but_soft_limit(self):
+        user_service_config = RoutingConfigMock(
+            username = self.username,
+            organization = None,
+            service_type = self.service_type,
+            monthly_quota = 1000,
+            period_end_date = datetime.today(),
+            soft_limit = True
+        )
+        redis_conn = MockRedis()
+        redis_conn.zincrby(self.redis_key, self.period_end_date.day, 1001)
+        checker = QuotaChecker(user_service_config, redis_conn)
+        assert checker.check() == True

server/lib/python/cartodb_services/test/metrics/test_user.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+from cartodb_services.metrics import UserMetricsService
+import datetime
+from mockredis import MockRedis
+
+class UserGeocoderConfig(object):
+
+    def __init__(self, **kwargs):
+        self.__dict__ = kwargs
+
+
+class TestUserMetricsService(TestCase):
+
+    def setUp(self):
+        user_geocoder_config = UserGeocoderConfig(
+            username = 'my_test_user',
+            organization = None,
+            period_end_date = datetime.date.today()
+        )
+        redis_conn = MockRedis()
+        self.user_metrics_service = UserMetricsService(user_geocoder_config, redis_conn)
+
+
+    def test_routing_used_quota_zero_when_no_usage(self):
+        assert self.user_metrics_service.used_quota(UserMetricsService.SERVICE_MAPZEN_ROUTING, datetime.date.today()) == 0
+
+    def test_routing_used_quota_counts_usages(self):
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'success_responses')
+        self.user_metrics_service.increment_service_use(UserMetricsService.SERVICE_MAPZEN_ROUTING, 'empty_responses')
+        assert self.user_metrics_service.used_quota('routing_mapzen', datetime.date.today()) == 2