Merge pull request #484 from CartoDB/development

Release server 0.30.3

NEWS.md

@@ -1,3 +1,43 @@
+
+March 16th, 2018
+================
+* Version `0.30.3` of server side
+    * Fix problem with invalid Mapbox isolines
+
+March 14th, 2018
+================
+* Version `0.17.4` of the python library
+    * Fix bug with previous version when checking quotas
+* Version `0.17.3` of the python library
+    * Fix bug with Mapbox routing not using the proper quota value
+
+February 22th, 2018
+==================
+* Version `0.17.2` of the python library
+    * Fix bug with Mapbox isolines not stopping at the seacoast
+
+February 27th, 2018
+==================
+* Version `0.17.1` of the python library
+    * Fix bug when the mapzen credentials are not in the db config and we keep getting them
+
+February 22th, 2018
+==================
+* Version `0.17.0` of the python library
+    * Change default provider to Mapbox
+    * Remove the obligatory nature of the Mapzen configuration due to its deprecation as provider
+
+February 13th, 2018
+==================
+* Version `0.16.7` of the python library
+    * Pick the first result when Mapbox geocoder returns multiple results #462
+    * Normalize input for Mapbox geocoder #462
+
+February 12th, 2018
+==================
+* Version `0.16.6` of the python library
+    * Using Mapbox permanent geocoder #455
+
 February 5th, 2018
 ==================
 * Version `0.16.5` of the python library

client/test/expected/00_installation_test.out

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

client/test/sql/00_installation_test.sql

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

doc/geocoding_functions.md

@@ -4,7 +4,7 @@ The [geocoder](https://carto.com/data/geocoder-api/) functions allow you to matc
 
 _**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
 
-Here is an example of how to geocode a single country:
+The following example displays how to geocode a single country:
 
 ```bash
 https://{username}.carto.com/api/v2/sql?q=SELECT cdb_geocode_admin0_polygon('USA')&api_key={api_key}
@@ -312,7 +312,7 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_ipaddress_point('102.23.34
 
 ## Street-Level Geocoder
 
-This function geocodes your data into a point geometry for a street address. CARTO uses several different service providers for street-level geocoding, depending on your platform. If you access CARTO on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [HERE geocoding services](https://developer.here.com/rest-apis/documentation/geocoder/topics/quick-start.html). Additional service providers will be implemented in the future.
+This function geocodes your data into a point geometry for a street address. CARTO uses several different service providers for street-level geocoding, depending on your platform. If you access CARTO on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [Mapbox geocoding services](https://www.mapbox.com/). [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account._.
 
 **This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) for details and recommendations about quota consumption.
 

doc/isoline_functions.md

@@ -10,7 +10,7 @@ You can use the isoline functions to retrieve, for example, isochrone lines from
 https://{username}.carto.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT  the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 600, 900]::integer[])&api_key={api_key}
 ```
 
-The following functions provide an isoline generator service, based on time or distance. This service uses the isolines service defined for your account. The default service limits the usage of displayed polygons represented on top of [HERE](https://developer.here.com/coverage-info) maps.
+The following functions provide an isoline generator service, based on time or distance. This service uses the isolines service defined for your account. The default service limits the usage of displayed polygons represented on top of [Mapbox](https://www.mapbox.com/) maps.
 
 ## cdb_isodistance(_source geometry, mode text, range integer[], [options text[]]_)
 

doc/overview.md

@@ -4,7 +4,7 @@ By using CARTO libraries and the SQL API, you can apply location data services t
 
 **Note:** Based on your account plan, some of these data services are subject to different [quota limitations](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-information).
 
-_The Data Services API is collaborating with [Mapzen](https://mapzen.com/), and several other geospatial service providers, in order to supply the best location data services from within our CARTO Engine._
+_In order to supply the best location data services from within our CARTO Engine, the Data Services API collaborates with [Mapbox](https://www.mapbox.com/) and several other geospatial service providers. [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account._
 
 ## Data Services Integration
 

doc/quota_information.md

@@ -59,9 +59,9 @@ Result:
 ```sql
     service     | monthly_quota | used_quota | soft_limit |     provider
 ----------------+---------------+------------+------------+------------------
- isolines       |           100 |          0 | f          | mapzen
- hires_geocoder |           100 |          0 | f          | mapzen
- routing        |            50 |          0 | f          | mapzen
+ isolines       |           100 |          0 | f          | mapbox
+ hires_geocoder |           100 |          0 | f          | mapbox
+ routing        |            50 |          0 | f          | mapbox
  observatory    |             0 |          0 | f          | data observatory
 (4 rows)
 
@@ -100,7 +100,7 @@ Suppose you want to geocode a whole table. In order to check that you have enoug
 SELECT COUNT(*) FROM {tablename} WHERE {street_name_column} IS NOT NULL;
 ```
 
-Result: here's a sample result of 10000 records:
+Result: A sample result of 10000 records:
 
 ```sql
  count

server/extension/cdb_dataservices_server--0.30.2--0.30.3.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.30.3--0.30.2.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.30.2'
+default_version = '0.30.3'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/80_isolines_helper.sql

@@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
         # -- TODO encapsulate this block into a func/method
         locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
         wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
-        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
         multipolygon = plpy.execute(sql, 1)[0]['geom']
       else:
         multipolygon = None

server/extension/test/expected/00_install_test.out

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

server/extension/test/sql/00_install_test.sql

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

server/lib/python/cartodb_services/README.md

@@ -24,7 +24,7 @@ NOTE: a system installation is required at present because the library is meant
 
 
 ## Running the unit tests
-Just run `nosetests test/`
+Just run `MAPBOX_API_KEY=xxx nosetests test/`
 ```shell
 $ nosetests test/
 ......................................................................................................

server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py

@@ -8,11 +8,12 @@ from mapbox import Geocoder
 from cartodb_services.metrics import Traceable
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools.qps import qps_retry
+from cartodb_services.tools.normalize import normalize
 
 GEOCODER_NAME = 'geocoder_name'
 EPHEMERAL_GEOCODER = 'mapbox.places'
 PERMANENT_GEOCODER = 'mapbox.places-permanent'
-DEFAULT_GEOCODER = EPHEMERAL_GEOCODER
+DEFAULT_GEOCODER = PERMANENT_GEOCODER
 
 ENTRY_FEATURES = 'features'
 ENTRY_CENTER = 'center'
@@ -32,17 +33,23 @@ class MapboxGeocoder(Traceable):
         self._token = token
         self._logger = logger
         self._geocoder_name = service_params.get(GEOCODER_NAME,
-                                                 EPHEMERAL_GEOCODER)
+                                                 DEFAULT_GEOCODER)
         self._geocoder = Geocoder(access_token=self._token,
                                   name=self._geocoder_name)
 
     def _parse_geocoder_response(self, response):
         json_response = json.loads(response)
 
-        if json_response and json_response[ENTRY_FEATURES]:
-            feature = json_response[ENTRY_FEATURES][0]
+        # If Mapbox returns more that one result, take the first one
+        if json_response:
+            if type(json_response) == list:
+                json_response = json_response[0]
 
-            return self._extract_lng_lat_from_feature(feature)
+            if json_response[ENTRY_FEATURES]:
+                feature = json_response[ENTRY_FEATURES][0]
+                return self._extract_lng_lat_from_feature(feature)
+            else:
+                return []
         else:
             return []
 
@@ -61,11 +68,11 @@ class MapboxGeocoder(Traceable):
     def geocode(self, searchtext, city=None, state_province=None,
                 country=None):
         if searchtext and searchtext.strip():
-            address = [searchtext]
+            address = [normalize(searchtext)]
             if city:
-                address.append(city)
+                address.append(normalize(city))
             if state_province:
-                address.append(state_province)
+                address.append(normalize(state_province))
         else:
             return []
 

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -4,6 +4,7 @@ Uses the Mapbox Time Matrix service.
 '''
 
 import json
+from cartodb_services.tools import Coordinate
 from cartodb_services.tools.spherical import (get_angles,
                                               calculate_dest_location)
 from cartodb_services.mapbox.matrix_client import (validate_profile,
@@ -11,7 +12,9 @@ from cartodb_services.mapbox.matrix_client import (validate_profile,
                                                    PROFILE_WALKING,
                                                    PROFILE_DRIVING,
                                                    PROFILE_CYCLING,
-                                                   ENTRY_DURATIONS)
+                                                   ENTRY_DURATIONS,
+                                                   ENTRY_DESTINATIONS,
+                                                   ENTRY_LOCATION)
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
@@ -53,6 +56,7 @@ class MapboxIsolines():
             return []
 
         costs = [None] * number_of_angles
+        destinations = [None] * number_of_angles
 
         for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
             if cost:
@@ -60,7 +64,11 @@ class MapboxIsolines():
             else:
                 costs[idx] = isorange
 
-        return costs
+        for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
+            destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
+                                           destination[ENTRY_LOCATION][1])
+
+        return costs, destinations
 
     def calculate_isochrone(self, origin, time_ranges,
                             profile=DEFAULT_PROFILE):
@@ -122,10 +130,12 @@ class MapboxIsolines():
             # NOTE: sometimes it cannot calculate the cost and returns None.
             #   Just assume isorange and stop the calculations there
 
-            costs = cost_method(origin=origin, targets=location_estimates,
-                                isorange=isorange, profile=profile,
-                                unit_factor=unit_factor,
-                                number_of_angles=number_of_angles)
+            costs, destinations = cost_method(origin=origin,
+                                              targets=location_estimates,
+                                              isorange=isorange,
+                                              profile=profile,
+                                              unit_factor=unit_factor,
+                                              number_of_angles=number_of_angles)
 
             if not costs:
                 continue
@@ -152,8 +162,8 @@ class MapboxIsolines():
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c != isorange:
-                location_estimates_filtered.append(location_estimates[i])
+            if c != isorange and c < isorange * (1 + tolerance):
+                location_estimates_filtered.append(destinations[i])
 
         return location_estimates_filtered
 

server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py

@@ -30,6 +30,8 @@ VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
                   PROFILE_WALKING]
 
 ENTRY_DURATIONS = 'durations'
+ENTRY_DESTINATIONS = 'destinations'
+ENTRY_LOCATION = 'location'
 
 
 def validate_profile(profile):

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -136,7 +136,7 @@ class RoutingConfig(ServiceConfig):
     ROUTING_PROVIDER_KEY = 'routing_provider'
     MAPZEN_PROVIDER = 'mapzen'
     MAPBOX_PROVIDER = 'mapbox'
-    DEFAULT_PROVIDER = MAPZEN_PROVIDER
+    DEFAULT_PROVIDER = MAPBOX_PROVIDER
     QUOTA_KEY = 'mapzen_routing_quota'
     SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
     METRICS_LOG_KEY = 'routing_log_path'
@@ -147,11 +147,13 @@ class RoutingConfig(ServiceConfig):
         self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY]
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
-        self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._mapzen_service_params = self._db_config.mapzen_routing_service_params
-        self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
-        self._mapbox_service_params = self._db_config.mapbox_routing_service_params
-        self._set_monthly_quota()
+        if self._routing_provider == self.MAPZEN_PROVIDER:
+            self._mapzen_api_key = self._db_config.mapzen_routing_api_key
+            self._mapzen_service_params = self._db_config.mapzen_routing_service_params
+        elif self._routing_provider == self.MAPBOX_PROVIDER:
+            self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
+            self._mapbox_service_params = self._db_config.mapbox_routing_service_params
+        self._routing_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
@@ -190,9 +192,13 @@ class RoutingConfig(ServiceConfig):
     def mapbox_service_params(self):
         return self._mapbox_service_params
 
+    @property
+    def routing_quota(self):
+        return self._routing_quota
+
     @property
     def monthly_quota(self):
-        return self._monthly_quota
+        return self._routing_quota
 
     @property
     def period_end_date(self):
@@ -202,9 +208,6 @@ class RoutingConfig(ServiceConfig):
     def soft_limit(self):
         return self._soft_limit
 
-    def _set_monthly_quota(self):
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-
     def _set_soft_limit(self):
         if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_limit = True
@@ -226,7 +229,7 @@ class IsolinesRoutingConfig(ServiceConfig):
     MAPZEN_PROVIDER = 'mapzen'
     MAPBOX_PROVIDER = 'mapbox'
     HEREMAPS_PROVIDER = 'heremaps'
-    DEFAULT_PROVIDER = MAPZEN_PROVIDER
+    DEFAULT_PROVIDER = MAPBOX_PROVIDER
     METRICS_LOG_KEY = 'isolines_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -391,7 +394,7 @@ class GeocoderConfig(ServiceConfig):
     USERNAME_KEY = 'username'
     ORGNAME_KEY = 'orgname'
     PERIOD_END_DATE = 'period_end_date'
-    DEFAULT_PROVIDER = MAPZEN_GEOCODER
+    DEFAULT_PROVIDER = MAPBOX_GEOCODER
     METRICS_LOG_KEY = 'geocoder_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
@@ -576,50 +579,51 @@ class ServicesDBConfig:
         heremaps_conf_json = self._get_conf('heremaps_conf')
         if not heremaps_conf_json:
             raise ConfigException('Here maps configuration missing')
-        else:
-            heremaps_conf = json.loads(heremaps_conf_json)
-            self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
-            self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
-            self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
-                'geocoder_cost_per_hit']
-            self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
-            self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
-            self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
-            self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
+
+        heremaps_conf = json.loads(heremaps_conf_json)
+        self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
+        self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
+        self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
+            'geocoder_cost_per_hit']
+        self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
+        self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
+        self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
+        self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
 
     def _get_mapzen_config(self):
         mapzen_conf_json = self._get_conf('mapzen_conf')
+        # We dont use mapzen anymore so we don't need to check for its configuration
         if not mapzen_conf_json:
-            raise ConfigException('Mapzen configuration missing')
-        else:
-            mapzen_conf = json.loads(mapzen_conf_json)
-            self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
-            self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
-            self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
-            self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
-            self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
-            self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
-            self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
-            self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
-            self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
-            self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
+            return
+
+        mapzen_conf = json.loads(mapzen_conf_json)
+        self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
+        self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
+        self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
+        self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
+        self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
+        self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
+        self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
+        self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
+        self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
+        self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
 
     def _get_mapbox_config(self):
         mapbox_conf_json = self._get_conf('mapbox_conf')
         if not mapbox_conf_json:
             raise ConfigException('Mapbox configuration missing')
-        else:
-            mapbox_conf = json.loads(mapbox_conf_json)
-            self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
-            self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
-            self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
-            self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
-            self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
-            self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
-            self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
-            self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys']
-            self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
-            self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
+
+        mapbox_conf = json.loads(mapbox_conf_json)
+        self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
+        self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
+        self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
+        self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
+        self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
+        self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
+        self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
+        self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys']
+        self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
+        self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
 
     def _get_data_observatory_config(self):
         do_conf_json = self._get_conf('data_observatory_conf')

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -122,7 +122,7 @@ class QuotaChecker:
             return False
 
     def __check_routing_quota(self):
-        user_quota = self._user_service_config.monthly_quota
+        user_quota = self._user_service_config.routing_quota
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)

server/lib/python/cartodb_services/cartodb_services/tools/normalize.py

@@ -0,0 +1,3 @@
+def normalize(str_input):
+    return str_input.replace('"', '"') \
+                    .replace(';', ',')

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.16.5',
+    version='0.17.4',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/credentials.py

@@ -0,0 +1,6 @@
+import os
+
+def mapbox_api_key():
+    """Returns Mapbox API key. Requires setting MAPBOX_API_KEY environment variable."""
+    return os.environ['MAPBOX_API_KEY']
+

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -301,7 +301,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
         orgname = None
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 1000
+        assert config.routing_quota == 1000
 
     def test_org_quota_overrides_user_quota(self):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
@@ -315,7 +315,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
 
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 5000
+        assert config.routing_quota == 5000
 
     def test_should_have_soft_limit_false_by_default(self):
         orgname = None

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -30,7 +30,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -43,7 +43,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -61,7 +61,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = False
         )
@@ -75,7 +75,7 @@ class TestQuotaChecker(TestCase):
             username = self.username,
             organization = None,
             service_type = self.service_type,
-            monthly_quota = 1000,
+            routing_quota = 1000,
             period_end_date = datetime.today(),
             soft_limit = True
         )

server/lib/python/cartodb_services/test/test_mapboxgeocoder.py

@@ -2,8 +2,8 @@ import unittest
 from mock import Mock
 from cartodb_services.mapbox import MapboxGeocoder
 from cartodb_services.tools.exceptions import ServiceException
+from credentials import mapbox_api_key
 
-VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
 INVALID_TOKEN = 'invalid_token'
 VALID_ADDRESS = 'Calle Siempreviva 3, Valladolid'
 WELL_KNOWN_LONGITUDE = -4.730947
@@ -12,7 +12,7 @@ WELL_KNOWN_LATITUDE = 41.668654
 
 class MapboxGeocoderTestCase(unittest.TestCase):
     def setUp(self):
-        self.geocoder = MapboxGeocoder(token=VALID_TOKEN, logger=Mock())
+        self.geocoder = MapboxGeocoder(token=mapbox_api_key(), logger=Mock())
 
     def test_invalid_token(self):
         invalid_geocoder = MapboxGeocoder(token=INVALID_TOKEN, logger=Mock())
@@ -35,6 +35,11 @@ class MapboxGeocoderTestCase(unittest.TestCase):
 
         assert place
 
+    def test_odd_characters(self):
+        place = self.geocoder.geocode(searchtext='Barcelona; "Spain"')
+
+        assert place
+
     def test_empty_request(self):
         place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None)
 

server/lib/python/cartodb_services/test/test_mapboxisoline.py

@@ -7,15 +7,15 @@ from cartodb_services.mapbox.routing import MapboxRouting
 from cartodb_services.tools import Coordinate
 from cartodb_services.tools.coordinates import (validate_coordinates,
                                                 marshall_coordinates)
+from credentials import mapbox_api_key
 
-VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
 VALID_ORIGIN = Coordinate(-73.989, 40.733)
 
 
 class MapboxIsolinesTestCase(unittest.TestCase):
 
     def setUp(self):
-        matrix_client = MapboxMatrixClient(token=VALID_TOKEN, logger=Mock())
+        matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock())
         self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock())
 
     def test_calculate_isochrone(self):

server/lib/python/cartodb_services/test/test_mapboxmatrix.py

@@ -4,8 +4,8 @@ from cartodb_services.mapbox import MapboxMatrixClient
 from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools import Coordinate
+from credentials import mapbox_api_key
 
-VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
 INVALID_TOKEN = 'invalid_token'
 VALID_ORIGIN = Coordinate(-73.989, 40.733)
 VALID_TARGET = Coordinate(-74, 40.733)
@@ -22,7 +22,7 @@ INVALID_PROFILE = 'invalid_profile'
 
 class MapboxMatrixTestCase(unittest.TestCase):
     def setUp(self):
-        self.matrix_client = MapboxMatrixClient(token=VALID_TOKEN,
+        self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),
                                                 logger=Mock())
 
     def test_invalid_profile(self):

server/lib/python/cartodb_services/test/test_mapboxrouting.py

@@ -4,8 +4,8 @@ from cartodb_services.mapbox import MapboxRouting
 from cartodb_services.mapbox.routing import DEFAULT_PROFILE
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools import Coordinate
+from credentials import mapbox_api_key
 
-VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
 INVALID_TOKEN = 'invalid_token'
 VALID_WAYPOINTS = [Coordinate(-73.989, 40.733), Coordinate(-74, 40.733)]
 NUM_WAYPOINTS_MAX = 25
@@ -31,7 +31,7 @@ WELL_KNOWN_LENGTH = 1317.9
 
 class MapboxRoutingTestCase(unittest.TestCase):
     def setUp(self):
-        self.routing = MapboxRouting(token=VALID_TOKEN, logger=Mock())
+        self.routing = MapboxRouting(token=mapbox_api_key(), logger=Mock())
 
     def test_invalid_profile(self):
         with self.assertRaises(ValueError):