Merge pull request #484 from CartoDB/development

Release server 0.30.3

NEWS.md

@@ -1,3 +1,26 @@
+
+March 16th, 2018
+================
+* Version `0.30.3` of server side
+    * Fix problem with invalid Mapbox isolines
+
+March 14th, 2018
+================
+* Version `0.17.4` of the python library
+    * Fix bug with previous version when checking quotas
+* Version `0.17.3` of the python library
+    * Fix bug with Mapbox routing not using the proper quota value
+
+February 22th, 2018
+==================
+* Version `0.17.2` of the python library
+    * Fix bug with Mapbox isolines not stopping at the seacoast
+
+February 27th, 2018
+==================
+* Version `0.17.1` of the python library
+    * Fix bug when the mapzen credentials are not in the db config and we keep getting them
+
 February 22th, 2018
 ==================
 * Version `0.17.0` of the python library

server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.30.2'
+default_version = '0.30.3'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/80_isolines_helper.sql

@@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
         # -- TODO encapsulate this block into a func/method
         locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
         wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
-        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
         multipolygon = plpy.execute(sql, 1)[0]['geom']
       else:
         multipolygon = None

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -4,6 +4,7 @@ Uses the Mapbox Time Matrix service.
 '''
 
 import json
+from cartodb_services.tools import Coordinate
 from cartodb_services.tools.spherical import (get_angles,
                                               calculate_dest_location)
 from cartodb_services.mapbox.matrix_client import (validate_profile,
@@ -11,7 +12,9 @@ from cartodb_services.mapbox.matrix_client import (validate_profile,
                                                    PROFILE_WALKING,
                                                    PROFILE_DRIVING,
                                                    PROFILE_CYCLING,
-                                                   ENTRY_DURATIONS)
+                                                   ENTRY_DURATIONS,
+                                                   ENTRY_DESTINATIONS,
+                                                   ENTRY_LOCATION)
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
@@ -53,6 +56,7 @@ class MapboxIsolines():
             return []
 
         costs = [None] * number_of_angles
+        destinations = [None] * number_of_angles
 
         for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
             if cost:
@@ -60,7 +64,11 @@ class MapboxIsolines():
             else:
                 costs[idx] = isorange
 
-        return costs
+        for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
+            destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
+                                           destination[ENTRY_LOCATION][1])
+
+        return costs, destinations
 
     def calculate_isochrone(self, origin, time_ranges,
                             profile=DEFAULT_PROFILE):
@@ -122,10 +130,12 @@ class MapboxIsolines():
             # NOTE: sometimes it cannot calculate the cost and returns None.
             #   Just assume isorange and stop the calculations there
 
-            costs = cost_method(origin=origin, targets=location_estimates,
-                                isorange=isorange, profile=profile,
-                                unit_factor=unit_factor,
-                                number_of_angles=number_of_angles)
+            costs, destinations = cost_method(origin=origin,
+                                              targets=location_estimates,
+                                              isorange=isorange,
+                                              profile=profile,
+                                              unit_factor=unit_factor,
+                                              number_of_angles=number_of_angles)
 
             if not costs:
                 continue
@@ -152,8 +162,8 @@ class MapboxIsolines():
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c != isorange:
-                location_estimates_filtered.append(location_estimates[i])
+            if c != isorange and c < isorange * (1 + tolerance):
+                location_estimates_filtered.append(destinations[i])
 
         return location_estimates_filtered
 

server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py

@@ -30,6 +30,8 @@ VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
                   PROFILE_WALKING]
 
 ENTRY_DURATIONS = 'durations'
+ENTRY_DESTINATIONS = 'destinations'
+ENTRY_LOCATION = 'location'
 
 
 def validate_profile(profile):

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -147,11 +147,13 @@ class RoutingConfig(ServiceConfig):
         self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY]
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
-        self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._mapzen_service_params = self._db_config.mapzen_routing_service_params
-        self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
-        self._mapbox_service_params = self._db_config.mapbox_routing_service_params
-        self._set_monthly_quota()
+        if self._routing_provider == self.MAPZEN_PROVIDER:
+            self._mapzen_api_key = self._db_config.mapzen_routing_api_key
+            self._mapzen_service_params = self._db_config.mapzen_routing_service_params
+        elif self._routing_provider == self.MAPBOX_PROVIDER:
+            self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
+            self._mapbox_service_params = self._db_config.mapbox_routing_service_params
+        self._routing_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
@@ -190,9 +192,13 @@ class RoutingConfig(ServiceConfig):
     def mapbox_service_params(self):
         return self._mapbox_service_params
 
+    @property
+    def routing_quota(self):
+        return self._routing_quota
+
     @property
     def monthly_quota(self):
-        return self._monthly_quota
+        return self._routing_quota
 
     @property
     def period_end_date(self):
@@ -202,9 +208,6 @@ class RoutingConfig(ServiceConfig):
     def soft_limit(self):
         return self._soft_limit
 
-    def _set_monthly_quota(self):
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-
     def _set_soft_limit(self):
         if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_limit = True

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -122,7 +122,7 @@ class QuotaChecker:
             return False
 
     def __check_routing_quota(self):
-        user_quota = self._user_service_config.monthly_quota
+        user_quota = self._user_service_config.routing_quota
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.17.0',
+    version='0.17.4',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -301,7 +301,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
         orgname = None
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 1000
+        assert config.routing_quota == 1000
 
     def test_org_quota_overrides_user_quota(self):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
@@ -315,7 +315,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
 
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 5000
+        assert config.routing_quota == 5000
 
     def test_should_have_soft_limit_false_by_default(self):
         orgname = None

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -30,7 +30,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -43,7 +43,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -61,7 +61,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -75,7 +75,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = True
         )