Merge pull request #578 from CartoDB/development

Added Mapbox true isolines

.github/workflows/main.yml

@@ -0,0 +1,52 @@
+name: dataservices-api PR testing
+on: push
+jobs:
+  dataservices-api:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        pg_version: [10, 12]
+    env:
+        PG_VERSION: ${{ matrix.pg_version }}
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup gcloud authentication
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.GCS }}
+
+      - name: Pull base image
+        run: gcloud auth configure-docker && docker pull gcr.io/cartodb-on-gcp-ci-testing/cartodb-postgresql-base:${{ matrix.pg_version }}
+
+      - name: Checkout ci tools repository
+        uses: actions/checkout@v2
+        with:
+          repository: CartoDB/ci-tools
+          path: ci-tools
+          token: ${{ secrets.CARTOFANTE_PAT }}
+
+      - name: Copy ci files to root
+        run: cp ci-tools/repos/${{ github.event.repository.name }}/* .
+
+      - name: Start docker-compose services
+        run: docker-compose -f docker-compose.yaml up -d
+
+      - name: Install required python libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services"
+
+      - name: Run python library tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test"
+        env:
+           MAPBOX_API_KEY: ${{ secrets.MAPBOX_API_KEY }}
+           TOMTOM_API_KEY: ${{ secrets.TOMTOM_API_KEY }}
+           GEOCODIO_API_KEY: ${{ secrets.GEOCODIO_API_KEY }}
+        timeout-minutes: 5
+
+      - name: Run server tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck"
+        timeout-minutes: 5
+
+      - name: Run client tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck"
+        timeout-minutes: 5

NEWS.md

@@ -1,3 +1,19 @@
+Feb 12th, 2020
+==============
+* Version `0.23.0` of the Python library
+    * Added Mapbox true isolines
+* Version `0.37.0` of the server extension
+    * Added Mapbox true isolines
+
+Jan 20th, 2020
+==============
+* Version `0.22.0` of the Python library
+    * Added geocodio geocoder
+* Version `0.36.0` of the server extension
+    * Added geocodio geocoder
+* Version `0.28.0` of the client extension
+    * Added geocodio geocoder
+
 Oct 9th, 2019
 ==============
 * Version `0.27.0` of the client extension

README.md

@@ -40,6 +40,7 @@ Steps to deploy a new Data Services API version :
     # in dataservices-api repo root path:
     cd server/lib/python/cartodb_services && sudo pip install -r requirements.txt && sudo pip install . --upgrade
     ```
+    **CLOUD DEPLOY NOTE**: we were not installing automatically `requirements.txt`, so we fixed it in https://github.com/CartoDB/cartodb-platform/pull/6187 . Please, be aware that in some corner cases scenarios, rolling back to a previous version might require to manually force-install some dependency versions that were upgraded previously in this step.
 
 - Create a database to hold all the server part and a user for it
 
@@ -180,6 +181,15 @@ SELECT CDB_Conf_SetConf(
 );
 ```
 
+#### Geocod.io configuration
+
+```sql
+SELECT CDB_Conf_SetConf(
+    'geocodio_conf',
+    '{"geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}}'
+);
+```
+
 #### Data Observatory
 
 ```sql

client/carto-package.json

@@ -2,12 +2,11 @@
   "name": "dataservices-api-client-extension",
   "current_version": {
     "requires": {
-      "postgresql": "^10.0.0",
+      "postgresql": "^11.0.0",
-      "postgis": "^2.4.0.0",
+      "postgis": "^2.5.0.0",
-      "carto_postgresql_ext": "^0.23.0"
+      "carto_postgresql_ext": "^0.32.0"
     },
     "works_with": {
-      "dataservices-api-server-extension": "^0.35.1"
     }
   }
 }

client/cdb_dataservices_client--0.27.0--0.28.0.sql

@@ -0,0 +1,83 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+         RETURN ret;
+  END;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_geocodio_geocode_street_point (username, orgname, searchtext, city, state_province, country);
+
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text ,state_province text ,country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
+

client/cdb_dataservices_client--0.28.0--0.27.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.27.0'
+default_version = '0.28.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/renderer/interface.yaml

@@ -160,6 +160,17 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: cdb_geocodio_geocode_street_point
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
 - name: cdb_mapzen_geocode_street_point
   return_type: public.Geometry
   requires_permission: true

server/extension/cdb_dataservices_server--0.36.0--0.37.0.sql

@@ -0,0 +1,126 @@
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.37.0--0.36.0.sql

@@ -0,0 +1,129 @@
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.35.1'
+default_version = '0.37.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.35.1--0.36.0.sql

@@ -0,0 +1,197 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.geocodio_geocoder:
+      geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.geocodio import GeocodioGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using Geocodio')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    elif user_geocoder_config.geocodio_geocoder:
+      provider_function = "_cdb_bulk_geocodio_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+  from cartodb_services.geocodio import GeocodioBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  params = {'username': username, 'orgname': orgname, 'country_name': country_name}
+
+  with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
+      rv = plpy.execute(plan, [country_name], 1)
+      result = rv[0]["mypolygon"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode admin0 polygon')
+    finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.36.0--0.35.1.sql

@@ -0,0 +1,79 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_geocodio_geocode_street_point;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_geocodio_geocode_street_point;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point;

server/extension/sql/20_geocode_street.sql

@@ -30,6 +30,9 @@ RETURNS Geometry AS $$
     elif user_geocoder_config.tomtom_geocoder:
       tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
       return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.geocodio_geocoder:
+      geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
     else:
       raise Exception('Requested geocoder is not available')
 
@@ -105,6 +108,19 @@ RETURNS Geometry AS $$
 
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.tools import LegacyServiceManager
@@ -302,3 +318,49 @@ RETURNS Geometry AS $$
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.geocodio import GeocodioGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using Geocodio')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/21_bulk_geocode_street.sql

@@ -32,6 +32,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
       provider_function = "_cdb_bulk_tomtom_geocode_street_point";
     elif user_geocoder_config.mapbox_geocoder:
       provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    elif user_geocoder_config.geocodio_geocoder:
+      provider_function = "_cdb_bulk_geocodio_geocode_street_point";
     else:
       raise Exception('Requested geocoder is not available')
 
@@ -96,3 +98,19 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+  from cartodb_services.geocodio import GeocodioBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/80_isolines_helper.sql

@@ -132,7 +132,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
    options text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
   from cartodb_services.tools import ServiceManager
-  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox import MapboxIsolines
   from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
   from cartodb_services.tools import Coordinate
   from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
@@ -144,8 +144,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   service_manager.assert_within_limits()
 
   try:
-    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
-    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
 
     if source:
       lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -169,7 +168,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
         # -- TODO encapsulate this block into a func/method
         locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
         wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
-        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
         multipolygon = plpy.execute(sql, 1)[0]['geom']
       else:
         multipolygon = None
@@ -322,7 +321,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
    options text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
   from cartodb_services.tools import ServiceManager
-  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox import MapboxIsolines
   from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
   from cartodb_services.tools import Coordinate
   from cartodb_services.tools.coordinates import coordinates_to_polygon
@@ -335,8 +334,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   service_manager.assert_within_limits()
 
   try:
-    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
-    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
 
     if source:
       lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']

server/extension/test/expected/00_install_test.out

@@ -39,7 +39,13 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
  cdb_conf_setconf 
 ------------------
  

server/extension/test/sql/00_install_test.sql

@@ -16,7 +16,8 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
-SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["isolines_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -2,10 +2,12 @@ from google import GoogleMapsBulkGeocoder
 from here import HereMapsBulkGeocoder
 from tomtom import TomTomBulkGeocoder
 from mapbox import MapboxBulkGeocoder
+from geocodio import GeocodioBulkGeocoder
 
 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
     'google': GoogleMapsBulkGeocoder,
     'heremaps': HereMapsBulkGeocoder,
     'tomtom': TomTomBulkGeocoder,
-    'mapbox': MapboxBulkGeocoder
+    'mapbox': MapboxBulkGeocoder,
+    'geocodio': GeocodioBulkGeocoder,
 }

server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py

@@ -0,0 +1,2 @@
+from geocoder import GeocodioGeocoder
+from bulk_geocoder import GeocodioBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/geocodio/bulk_geocoder.py

@@ -0,0 +1,76 @@
+import requests
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.geocodio import GeocodioGeocoder
+from iso3166 import countries
+from cartodb_services.tools.country import country_to_iso3
+
+
+class GeocodioBulkGeocoder(GeocodioGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 100  # Setting an upper limit (not stated in the documentation)
+    MIN_BATCHED_SEARCH = 0
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    def __init__(self, token, logger, service_params=None):
+        GeocodioGeocoder.__init__(self, token, logger, service_params)
+
+        self.connect_timeout = self.CONNECT_TIMEOUT
+        self.read_timeout = self.READ_TIMEOUT
+        self.max_retries = self.MAX_RETRIES
+
+        if service_params is not None:
+            self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
+            self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
+            self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
+
+        self.session = requests.Session()
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            elements = self._encoded_elements(search)
+            result = self.geocode_meta(*elements)
+
+            if result:
+                results.append((search[0], result[0], result[1]))
+            else:
+                results.append((search[0], None, None))
+
+        return results
+
+    def _encoded_elements(self, search):
+        (search_id, address, city, state, country) = search
+        address = address.encode('utf-8') if address else None
+        city = city.encode('utf-8') if city else None
+        state = state.encode('utf-8') if state else None
+        country = self._country_code(country) if country else None
+        return address, city, state, country
+
+    def _batch_geocode(self, searches):
+        if len(searches) == 1:
+            return self._serial_geocode(searches)
+        else:
+            frees = []
+            for search in searches:
+                elements = self._encoded_elements(search)
+                free = ', '.join([elem for elem in elements if elem])
+                frees.append(free)
+
+            full_results = self.geocode_free_text_meta(frees)
+            results = []
+            for s, r in zip(searches, full_results):
+                results.append((s[0], r[0], r[1]))
+            return results
+
+    def _country_code(self, country):
+        country_iso3166 = country
+        country_iso3 = country_to_iso3(country)
+        if country_iso3:
+            country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+        return country_iso3166
+

server/lib/python/cartodb_services/cartodb_services/geocodio/geocoder.py

@@ -0,0 +1,126 @@
+from geocodio import GeocodioClient
+from geocodio.exceptions import GeocodioAuthError, GeocodioServerError, GeocodioDataError, GeocodioError
+
+from cartodb_services.tools.qps import qps_retry
+from cartodb_services.metrics import Traceable
+from cartodb_services.geocoder import EMPTY_RESPONSE, geocoder_metadata
+from cartodb_services.tools.exceptions import ServiceException
+
+
+RELEVANCE_BY_LOCATION_TYPE = {
+    'rooftop': 1,
+    'point': 0.9,
+    'range_interpolation': 0.8,
+    'nearest_rooftop_match': 0.7,
+    'intersection': 0.6,
+    'street_center': 0.5,
+    'place': 0.4,
+    'state': 0.1,
+}
+
+
+class GeocodioGeocoder(Traceable):
+    '''
+    Python wrapper for the Geocodio Geocoder service.
+    '''
+
+    def __init__(self, token, logger, service_params=None):
+        service_params = service_params or {}
+        self._token = token
+        self._logger = logger
+
+        self._geocoder = GeocodioClient(self._token)
+
+    def _validate_input(self, searchtext, city=None, state_province=None,
+                        country=None):
+        if searchtext and searchtext.strip():
+            return True
+        elif city:
+            return True
+        elif state_province:
+            return True
+
+        return False
+
+    @qps_retry(qps=15, provider='geocodio')
+    def geocode(self, searchtext, city=None, state_province=None,
+                country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                     country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    @qps_retry(qps=15, provider='geocodio')
+    def _geocode_meta(self, searchtext, city=None, state_province=None,
+                      country=None):
+        if not self._validate_input(searchtext, city, state_province, country):
+            return EMPTY_RESPONSE
+
+        try:
+            free_text_components = [searchtext, city, state_province, country]
+            req = '; '.join([c for c in free_text_components if c is not None and c.strip()])
+            response = self._geocoder.geocode(req)
+
+            return self._parse_geocoder_response(response)
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+    @qps_retry(qps=15)
+    def geocode_free_text_meta(self, free_searches, country=None):
+        """
+        :param free_searches: Free text searches
+        :return: list of [x, y] on success, [] on error
+        """
+        output = []
+
+        try:
+            if country:
+                free_searches = ['{s}, {country}'.format(s, country) for s in free_searches]
+
+            responses = self._geocoder.geocode(free_searches)
+
+            for response in responses:
+                output.append(self._parse_geocoder_response(response))
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+        return output
+
+    def _parse_geocoder_response(self, response):
+        if response is None or not response:
+            return EMPTY_RESPONSE
+
+        if response.get('results') is None or not response.get('results'):
+            return EMPTY_RESPONSE
+
+        if response.coords is None or not response.coords:
+            return EMPTY_RESPONSE
+
+        coords = [None, None]
+        accuracy = None
+        accuracy_type = None
+
+        accuracy = response.accuracy
+
+        if response.coords is not None and response.coords:
+            coords = [response.coords[1], response.coords[0]]
+
+        if response.get('results'):
+            accuracy_type = response.get('results')[0].get('accuracy_type')
+
+        metadata = geocoder_metadata(RELEVANCE_BY_LOCATION_TYPE.get(accuracy_type), response.accuracy, accuracy_type)
+
+        return [coords, metadata]

server/lib/python/cartodb_services/cartodb_services/geocodio/types.py

@@ -0,0 +1 @@
+GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'

server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py

@@ -2,4 +2,3 @@ from routing import MapboxRouting, MapboxRoutingResponse
 from geocoder import MapboxGeocoder
 from bulk_geocoder import MapboxBulkGeocoder
 from isolines import MapboxIsolines, MapboxIsochronesResponse
-from matrix_client import MapboxMatrixClient

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -1,171 +1,142 @@
-'''
-Python implementation for Mapbox services based isolines.
-Uses the Mapbox Time Matrix service.
-'''
-
 import json
+import requests
+from uritemplate import URITemplate
+
+from cartodb_services.tools.exceptions import ServiceException
+from cartodb_services.tools.qps import qps_retry
 from cartodb_services.tools import Coordinate
-from cartodb_services.tools.spherical import (get_angles,
+
-                                              calculate_dest_location)
+BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}')
-from cartodb_services.mapbox.matrix_client import (validate_profile,
+
-                                                   DEFAULT_PROFILE,
+PROFILE_DRIVING = 'driving'
-                                                   PROFILE_WALKING,
+PROFILE_CYCLING = 'cycling'
-                                                   PROFILE_DRIVING,
+PROFILE_WALKING = 'walking'
-                                                   PROFILE_CYCLING,
+DEFAULT_PROFILE = PROFILE_DRIVING
-                                                   ENTRY_DURATIONS,
+
-                                                   ENTRY_DESTINATIONS,
+MAX_TIME_RANGE = 60 * 60  # The maximum time that can be specified is 60 minutes.
-                                                   ENTRY_LOCATION)
+                          # https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
     PROFILE_CYCLING: 16.67,  # In m/s, assuming 60km/h max speed
-    PROFILE_DRIVING: 41.67  # In m/s, assuming 140km/h max speed
+    PROFILE_DRIVING: 38.89  # In m/s, assuming 140km/h max speed
 }
 
-DEFAULT_NUM_ANGLES = 24
+VALID_PROFILES = (PROFILE_DRIVING,
-DEFAULT_MAX_ITERS = 5
+                  PROFILE_CYCLING,
-DEFAULT_TOLERANCE = 0.1
+                  PROFILE_WALKING)
 
-MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES
+ENTRY_FEATURES = 'features'
-MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS
+ENTRY_GEOMETRY = 'geometry'
-MATRIX_TOLERANCE = DEFAULT_TOLERANCE
+ENTRY_COORDINATES = 'coordinates'
-
-UNIT_FACTOR_ISOCHRONE = 1.0
-UNIT_FACTOR_ISODISTANCE = 1000.0
-DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE
 
 
 class MapboxIsolines():
     '''
-    Python wrapper for Mapbox services based isolines.
+    Python wrapper for Mapbox based isolines.
     '''
 
-    def __init__(self, matrix_client, logger, service_params=None):
+    def __init__(self, apikey, logger, service_params=None):
         service_params = service_params or {}
-        self._matrix_client = matrix_client
+        self._apikey = apikey
         self._logger = logger
 
-    def _calculate_matrix_cost(self, origin, targets, isorange,
+    def _uri(self, origin, time_range, profile=DEFAULT_PROFILE):
-                               profile=DEFAULT_PROFILE,
+        uri = URITemplate(BASEURI).expand(apikey=self._apikey,
-                               unit_factor=UNIT_FACTOR_ISOCHRONE,
+                                          coordinates=origin,
-                               number_of_angles=MATRIX_NUM_ANGLES):
+                                          contours_minutes=time_range,
-        response = self._matrix_client.matrix([origin] + targets,
+                                          profile=profile)
-                                              profile)
+        return uri
+
+    def _validate_profile(self, profile):
+        if profile not in VALID_PROFILES:
+            raise ValueError('{profile} is not a valid profile. '
+                             'Valid profiles are: {valid_profiles}'.format(
+                                 profile=profile,
+                                 valid_profiles=', '.join(
+                                     [x for x in VALID_PROFILES])))
+
+    def _validate_time_ranges(self, time_ranges):
+        for time_range in time_ranges:
+            if time_range > MAX_TIME_RANGE:
+                raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format(
+                                    max_time_range=MAX_TIME_RANGE))
+
+    def _parse_coordinates(self, boundary):
+        coordinates = boundary.get(ENTRY_COORDINATES, [])
+        return [Coordinate(c[0], c[1]) for c in coordinates]
+
+    def _parse_isochrone_service(self, response):
         json_response = json.loads(response)
-        if not json_response:
-            return []
 
-        costs = [None] * number_of_angles
+        coordinates = []
-        destinations = [None] * number_of_angles
+        if json_response:
+            for feature in json_response[ENTRY_FEATURES]:
+                geometry = feature[ENTRY_GEOMETRY]
+                coordinates.append(self._parse_coordinates(geometry))
 
-        for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
+        return coordinates
-            if cost:
+
-                costs[idx] = cost * unit_factor
+    @qps_retry(qps=5, provider='mapbox')
+    def _calculate_isoline(self, origin, time_ranges,
+                           profile=DEFAULT_PROFILE):
+        self._validate_time_ranges(time_ranges)
+
+        origin = '{lon},{lat}'.format(lat=origin.latitude,
+                                      lon=origin.longitude)
+
+        time_ranges.sort()
+        time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges])
+
+        uri = self._uri(origin, time_ranges_seconds, profile)
+
+        try:
+            response = requests.get(uri)
+
+            if response.status_code == requests.codes.ok:
+                isolines = []
+                coordinates = self._parse_isochrone_service(response.text)
+                for t, c in zip(time_ranges, coordinates):
+                    isolines.append(MapboxIsochronesResponse(c, t))
+
+                return isolines
+            elif response.status_code == requests.codes.bad_request:
+                return []
+            elif response.status_code == requests.codes.unprocessable_entity:
+                return []
             else:
-                costs[idx] = isorange
+                raise ServiceException(response.status_code, response)
-
+        except requests.Timeout as te:
-        for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
+            # In case of timeout we want to stop the job because the server
-            destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
+            # could be down
-                                           destination[ENTRY_LOCATION][1])
+            self._logger.error('Timeout connecting to Mapbox isochrone service',
-
+                               te)
-        return costs, destinations
+            raise ServiceException('Error getting isochrone data from Mapbox',
+                                   None)
+        except requests.ConnectionError as ce:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to Mapbox isochrone service',
+                               exception=ce)
+            return []
 
     def calculate_isochrone(self, origin, time_ranges,
                             profile=DEFAULT_PROFILE):
-        validate_profile(profile)
+        self._validate_profile(profile)
 
-        max_speed = MAX_SPEEDS[profile]
+        return self._calculate_isoline(origin=origin,
-
+                                       time_ranges=time_ranges,
-        isochrones = []
+                                       profile=profile)
-        for time_range in time_ranges:
-            upper_rmax = max_speed * time_range  # an upper bound for the radius
-
-            coordinates = self.calculate_isoline(origin=origin,
-                                                 isorange=time_range,
-                                                 upper_rmax=upper_rmax,
-                                                 cost_method=self._calculate_matrix_cost,
-                                                 profile=profile,
-                                                 unit_factor=UNIT_FACTOR_ISOCHRONE,
-                                                 number_of_angles=MATRIX_NUM_ANGLES,
-                                                 max_iterations=MATRIX_MAX_ITERS,
-                                                 tolerance=MATRIX_TOLERANCE)
-            isochrones.append(MapboxIsochronesResponse(coordinates,
-                                                       time_range))
-        return isochrones
 
     def calculate_isodistance(self, origin, distance_range,
                               profile=DEFAULT_PROFILE):
-        validate_profile(profile)
+        self._validate_profile(profile)
 
         max_speed = MAX_SPEEDS[profile]
         time_range = distance_range / max_speed
 
-        return self.calculate_isochrone(origin=origin,
+        return self._calculate_isoline(origin=origin,
-                                        time_ranges=[time_range],
+                                       time_ranges=[time_range],
-                                        profile=profile)[0].coordinates
+                                       profile=profile)[0].coordinates
-
-    def calculate_isoline(self, origin, isorange, upper_rmax,
-                          cost_method=_calculate_matrix_cost,
-                          profile=DEFAULT_PROFILE,
-                          unit_factor=DEFAULT_UNIT_FACTOR,
-                          number_of_angles=DEFAULT_NUM_ANGLES,
-                          max_iterations=DEFAULT_MAX_ITERS,
-                          tolerance=DEFAULT_TOLERANCE):
-        # Formally, a solution is an array of {angle, radius, lat, lon, cost}
-        # with cardinality number_of_angles
-        # we're looking for a solution in which
-        # abs(cost - isorange) / isorange <= TOLERANCE
-
-        # Initial setup
-        angles = get_angles(number_of_angles)
-        rmax = [upper_rmax] * number_of_angles
-        rmin = [0.0] * number_of_angles
-        location_estimates = [calculate_dest_location(origin, a,
-                                                      upper_rmax / 2.0)
-                              for a in angles]
-
-        # Iterate to refine the first solution
-        for i in xrange(0, max_iterations):
-            # Calculate the "actual" cost for each location estimate.
-            # NOTE: sometimes it cannot calculate the cost and returns None.
-            #   Just assume isorange and stop the calculations there
-
-            costs, destinations = cost_method(origin=origin,
-                                              targets=location_estimates,
-                                              isorange=isorange,
-                                              profile=profile,
-                                              unit_factor=unit_factor,
-                                              number_of_angles=number_of_angles)
-
-            if not costs:
-                continue
-
-            errors = [(cost - isorange) / float(isorange) for cost in costs]
-            max_abs_error = max([abs(e) for e in errors])
-            if max_abs_error <= tolerance:
-                # good enough, stop there
-                break
-
-            # let's refine the solution, binary search
-            for j in xrange(0, number_of_angles):
-
-                if abs(errors[j]) > tolerance:
-                    if errors[j] > 0:
-                        rmax[j] = (rmax[j] + rmin[j]) / 2.0
-                    else:
-                        rmin[j] = (rmax[j] + rmin[j]) / 2.0
-
-                    location_estimates[j] = calculate_dest_location(origin,
-                                                                    angles[j],
-                                                                    (rmax[j] + rmin[j]) / 2.0)
-
-        # delete points that got None
-        location_estimates_filtered = []
-        for i, c in enumerate(costs):
-            if c != isorange and c < isorange * (1 + tolerance):
-                location_estimates_filtered.append(destinations[i])
-
-        return location_estimates_filtered
 
 
 class MapboxIsochronesResponse:

server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py

@@ -1,92 +0,0 @@
-'''
-Python client for the Mapbox Time Matrix service.
-'''
-
-import requests
-from cartodb_services.metrics import Traceable
-from cartodb_services.tools.coordinates import (validate_coordinates,
-                                                marshall_coordinates)
-from cartodb_services.tools.exceptions import ServiceException
-from cartodb_services.tools.qps import qps_retry
-
-BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
-           '{coordinates}'
-           '?access_token={token}'
-           '&sources=0'  # Set the first coordinate as source...
-           '&destinations=all')  # ...and the rest as destinations
-
-NUM_COORDINATES_MIN = 2  # https://www.mapbox.com/api-documentation/#matrix
-NUM_COORDINATES_MAX = 25  # https://www.mapbox.com/api-documentation/#matrix
-
-PROFILE_DRIVING_TRAFFIC = 'driving-traffic'
-PROFILE_DRIVING = 'driving'
-PROFILE_CYCLING = 'cycling'
-PROFILE_WALKING = 'walking'
-DEFAULT_PROFILE = PROFILE_DRIVING
-
-VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
-                  PROFILE_DRIVING,
-                  PROFILE_CYCLING,
-                  PROFILE_WALKING]
-
-ENTRY_DURATIONS = 'durations'
-ENTRY_DESTINATIONS = 'destinations'
-ENTRY_LOCATION = 'location'
-
-
-def validate_profile(profile):
-    if profile not in VALID_PROFILES:
-        raise ValueError('{profile} is not a valid profile. '
-                         'Valid profiles are: {valid_profiles}'.format(
-                             profile=profile,
-                             valid_profiles=', '.join(
-                                 [x for x in VALID_PROFILES])))
-
-
-class MapboxMatrixClient(Traceable):
-    '''
-    Python wrapper for the Mapbox Time Matrix service.
-    '''
-
-    def __init__(self, token, logger, service_params=None):
-        service_params = service_params or {}
-        self._token = token
-        self._logger = logger
-
-    def _uri(self, coordinates, profile=DEFAULT_PROFILE):
-        return BASEURI.format(profile=profile, coordinates=coordinates,
-                              token=self._token)
-
-    @qps_retry(qps=1)
-    def matrix(self, coordinates, profile=DEFAULT_PROFILE):
-        validate_profile(profile)
-        validate_coordinates(coordinates,
-                             NUM_COORDINATES_MIN, NUM_COORDINATES_MAX)
-
-        coords = marshall_coordinates(coordinates)
-
-        uri = self._uri(coords, profile)
-
-        try:
-            response = requests.get(uri)
-
-            if response.status_code == requests.codes.ok:
-                return response.text
-            elif response.status_code == requests.codes.bad_request:
-                return '{}'
-            elif response.status_code == requests.codes.unprocessable_entity:
-                return '{}'
-            else:
-                raise ServiceException(response.status_code, response)
-        except requests.Timeout as te:
-            # In case of timeout we want to stop the job because the server
-            # could be down
-            self._logger.error('Timeout connecting to Mapbox matrix service',
-                               te)
-            raise ServiceException('Error getting matrix data from Mapbox',
-                                   None)
-        except requests.ConnectionError as ce:
-            # Don't raise the exception to continue with the geocoding job
-            self._logger.error('Error connecting to Mapbox matrix service',
-                               exception=ce)
-            return '{}'

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -405,6 +405,8 @@ class GeocoderConfig(ServiceConfig):
     MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
     TOMTOM_GEOCODER = 'tomtom'
     TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
+    GEOCODIO_GEOCODER = 'geocodio'
+    GEOCODIO_GEOCODER_API_KEYS = 'geocodio_geocoder_api_keys'
     QUOTA_KEY = 'geocoding_quota'
     SOFT_LIMIT_KEY = 'soft_geocoding_limit'
     USERNAME_KEY = 'username'
@@ -437,6 +439,9 @@ class GeocoderConfig(ServiceConfig):
         elif self._geocoder_provider == self.TOMTOM_GEOCODER:
             if not self.tomtom_api_keys:
                 raise ConfigException("""TomTom config is not set up""")
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            if not self.geocodio_api_keys:
+                raise ConfigException("""Geocodio config is not set up""")
 
         return True
 
@@ -476,6 +481,10 @@ class GeocoderConfig(ServiceConfig):
             self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
             self._cost_per_hit = 0
             self._tomtom_service_params = db_config.tomtom_geocoder_service_params
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            self._geocodio_api_keys = db_config.geocodio_geocoder_api_keys
+            self._cost_per_hit = 0
+            self._geocodio_service_params = db_config.geocodio_geocoder_service_params
 
     @property
     def service_type(self):
@@ -489,6 +498,8 @@ class GeocoderConfig(ServiceConfig):
             return 'geocoder_tomtom'
         elif self._geocoder_provider == self.NOKIA_GEOCODER:
             return 'geocoder_here'
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            return 'geocoder_geocodio'
 
     @property
     def heremaps_geocoder(self):
@@ -510,6 +521,10 @@ class GeocoderConfig(ServiceConfig):
     def tomtom_geocoder(self):
         return self._geocoder_provider == self.TOMTOM_GEOCODER
 
+    @property
+    def geocodio_geocoder(self):
+        return self._geocoder_provider == self.GEOCODIO_GEOCODER
+
     @property
     def google_client_id(self):
         return self._google_maps_client_id
@@ -569,6 +584,14 @@ class GeocoderConfig(ServiceConfig):
     def tomtom_service_params(self):
         return self._tomtom_service_params
 
+    @property
+    def geocodio_api_keys(self):
+        return self._geocodio_api_keys
+
+    @property
+    def geocodio_service_params(self):
+        return self._geocodio_service_params
+
     @property
     def is_high_resolution(self):
         return True
@@ -600,6 +623,7 @@ class ServicesDBConfig:
         self._get_mapzen_config()
         self._get_mapbox_config()
         self._get_tomtom_config()
+        self._get_geocodio_config()
         self._get_data_observatory_config()
 
     def _get_server_config(self):
@@ -652,6 +676,9 @@ class ServicesDBConfig:
             raise ConfigException('Mapbox configuration missing')
 
         mapbox_conf = json.loads(mapbox_conf_json)
+
+        # Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
+        # to `isolines` to ensure retrocompatibility
         self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
         self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
         self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
@@ -679,6 +706,16 @@ class ServicesDBConfig:
             self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
             self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})
 
+    def _get_geocodio_config(self):
+        geocodio_conf_json = self._get_conf('geocodio_conf')
+        if not geocodio_conf_json:
+            raise ConfigException('Geocodio configuration missing')
+        else:
+            geocodio_conf = json.loads(geocodio_conf_json)
+            self._geocodio_geocoder_api_keys = geocodio_conf['geocoder']['api_keys']
+            self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
+            self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
+
     def _get_data_observatory_config(self):
         do_conf_json = self._get_conf('data_observatory_conf')
         if not do_conf_json:
@@ -848,6 +885,18 @@ class ServicesDBConfig:
     def tomtom_geocoder_service_params(self):
         return self._tomtom_geocoder_service_params
 
+    @property
+    def geocodio_geocoder_api_keys(self):
+        return self._geocodio_geocoder_api_keys
+
+    @property
+    def geocodio_geocoder_monthly_quota(self):
+        return self.geocodio_geocoder_quota
+
+    @property
+    def geocodio_geocoder_service_params(self):
+        return self._geocodio_geocoder_service_params
+
     @property
     def data_observatory_connection_str(self):
         return self._data_observatory_connection_str

server/lib/python/cartodb_services/cartodb_services/refactor/service/geocodio_geocoder_config.py

@@ -0,0 +1,128 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.geocodio.types import GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN
+
+
+class GeocodioGeocoderConfig(object):
+    """
+    Configuration needed to operate the Geocodio geocoder service.
+    """
+
+    def __init__(self,
+                 geocoding_quota,
+                 soft_geocoding_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 geocodio_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._geocoding_quota = geocoding_quota
+        self._soft_geocoding_limit = soft_geocoding_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._geocodio_api_keys = geocodio_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'geocoder_geocodio'
+
+    @property
+    def provider(self):
+        return 'geocodio'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def geocodio_api_key(self):
+        return round_robin(self._geocodio_api_keys, self._GD,
+                           GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+    # TODO: for BW compat, remove
+    @property
+    def google_geocoder(self):
+        return False
+
+
+class GeocodioGeocoderConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        geocodio_server_conf = self._server_conf.get('geocodio_conf')
+        geocodio_api_keys = geocodio_server_conf['geocoder']['api_keys']
+        geocodio_service_params = geocodio_server_conf['geocoder'].get('service', {})
+
+        geocoding_quota = self._get_quota()
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('geocoder_log_path', None)
+
+        return GeocodioGeocoderConfig(geocoding_quota,
+                                    soft_geocoding_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    geocodio_api_keys,
+                                    self._username,
+                                    self._orgname,
+                                    geocodio_service_params,
+                                    self._GD)
+
+    def _get_quota(self):
+        geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
+        if geocoding_quota is '':
+            return 0
+
+        return int(geocoding_quota)

server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py

@@ -92,6 +92,9 @@ class MapboxIsolinesConfigBuilder(object):
 
     def get(self):
         mapbox_server_conf = self._server_conf.get('mapbox_conf')
+
+        # Note: We are no longer using the Matrix API but we have avoided renaming the `matrix` parameter
+        # to `isolines` to ensure retrocompatibility
         mapbox_api_keys = mapbox_server_conf['matrix']['api_keys']
         mapbox_service_params = mapbox_server_conf['matrix'].get('service', {})
 

server/lib/python/cartodb_services/requirements.txt

@@ -7,6 +7,7 @@ rollbar==0.13.2
 requests==2.9.1
 rratelimit==0.0.4
 mapbox==0.14.0
+pygeocodio==0.11.1
 
 # Test
 mock==1.3.0

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.21.4',
+    version='0.23.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/credentials.py

@@ -9,3 +9,8 @@ def mapbox_api_key():
 def tomtom_api_key():
     """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable."""
     return os.environ['TOMTOM_API_KEY']
+
+
+def geocodio_api_key():
+    """Returns Geocodio API key. Requires setting GEOCODIO_API_KEY environment variable."""
+    return os.environ['GEOCODIO_API_KEY']

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
 
 class TestGeocoderUserConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -31,6 +31,9 @@ class TestGeocoderUserConfig(TestCase):
             elif geocoder_provider == 'tomtom':
                 assert geocoder_config.tomtom_geocoder is True
                 assert geocoder_config.geocoding_quota == 100
+            elif geocoder_provider == 'geocodio':
+                assert geocoder_config.geocodio_geocoder is True
+                assert geocoder_config.geocoding_quota == 100
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None
@@ -84,7 +87,7 @@ class TestGeocoderUserConfig(TestCase):
 
 class TestGeocoderOrgConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -113,6 +116,9 @@ class TestGeocoderOrgConfig(TestCase):
             elif geocoder_provider == 'tomtom':
                 assert geocoder_config.tomtom_geocoder is True
                 assert geocoder_config.geocoding_quota == 200
+            elif geocoder_provider == 'geocodio':
+                assert geocoder_config.geocodio_geocoder is True
+                assert geocoder_config.geocoding_quota == 200
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None

server/lib/python/cartodb_services/test/test_geocodiogeocoder.py

@@ -0,0 +1,268 @@
+import unittest
+from mock import Mock
+from cartodb_services.geocodio import GeocodioGeocoder
+from cartodb_services.geocodio import GeocodioBulkGeocoder
+from cartodb_services.tools.exceptions import ServiceException
+from credentials import geocodio_api_key
+
+INVALID_TOKEN = 'invalid_token'
+
+VALID_ADDRESS_1 = 'Lexington Ave; New York; US'
+VALID_ADDRESS_2 = 'E 14th St; New York; US'
+VALID_ADDRESS_3 = '652 Lombard Street; San Francisco; California; United States'
+
+VALID_SEARCH_TEXT_1='Lexington Ave'
+VALID_CITY_1='New York'
+VALID_STATE_PROVINCE_1='New York'
+VALID_COUNTRY_1='US'
+
+VALID_SEARCH_TEXT_2='E 14th St'
+VALID_CITY_2='New York'
+VALID_STATE_PROVINCE_2='New York'
+VALID_COUNTRY_2='US'
+
+VALID_SEARCH_TEXT_3='652 Lombard Street'
+VALID_CITY_3='San Francisco'
+VALID_STATE_PROVINCE_3='California'
+VALID_COUNTRY_3='United States'
+
+WELL_KNOWN_LONGITUDE_1 = -73.960
+WELL_KNOWN_LATITUDE_1 = 40.774
+WELL_KNOWN_LONGITUDE_2 = -73.983
+WELL_KNOWN_LATITUDE_2 = 40.731
+WELL_KNOWN_LONGITUDE_3 = -122.412
+WELL_KNOWN_LATITUDE_3 = 37.803207
+
+SEARCH_ID_1 = 1
+SEARCH_ID_2 = 2
+
+
+class GeocodioGeocoderTestCase(unittest.TestCase):
+    def setUp(self):
+        self.geocoder = GeocodioGeocoder(token=geocodio_api_key(), logger=Mock())
+        self.bulk_geocoder = GeocodioBulkGeocoder(token=geocodio_api_key(), logger=Mock())
+
+    ### NON BULK
+
+    def test_invalid_token(self):
+        invalid_geocoder = GeocodioGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder.geocode(VALID_ADDRESS_1)
+
+    def test_valid_requests(self):
+        place = self.geocoder.geocode(VALID_ADDRESS_1)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.geocoder.geocode(VALID_ADDRESS_2)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.geocoder.geocode(VALID_ADDRESS_3)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_components(self):
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_1,
+                                      city=VALID_CITY_1,
+                                      state_province=VALID_STATE_PROVINCE_1,
+                                      country=VALID_COUNTRY_1)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_2,
+                                      city=VALID_CITY_2,
+                                      state_province=VALID_STATE_PROVINCE_2,
+                                      country=VALID_COUNTRY_2)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_3,
+                                      city=VALID_CITY_3,
+                                      state_province=VALID_STATE_PROVINCE_3,
+                                      country=VALID_COUNTRY_3)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_namedplace(self):
+        place = self.geocoder.geocode(searchtext='New York')
+
+        assert place
+
+    def test_valid_request_namedplace2(self):
+        place = self.geocoder.geocode(searchtext='New York', country='us')
+
+        assert place
+
+    def test_odd_characters(self):
+        place = self.geocoder.geocode(searchtext='New York; "USA"')
+
+        assert place
+
+    def test_empty_request(self):
+        place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None)
+
+        assert place == []
+
+    def test_empty_search_text_request(self):
+        place = self.geocoder.geocode(searchtext='     ', country='us', city=None, state_province="")
+
+        assert place == []
+
+    def test_unknown_place_request(self):
+        place = self.geocoder.geocode(searchtext='[unknown]', country='ch', state_province=None, city=None)
+
+        assert place == []
+
+    ### BULK ONE
+
+    def test_invalid_token_bulk_one(self):
+        invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
+
+    def test_valid_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_2, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_3, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_components_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_3, VALID_CITY_3, VALID_STATE_PROVINCE_3, VALID_COUNTRY_3)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_namedplace_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None)])
+
+        assert place
+
+    def test_valid_request_namedplace2_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None)])
+
+        assert place
+
+    def test_odd_characters_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None)])
+
+        assert place
+
+    def test_empty_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None)])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    def test_empty_search_text_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '     ', 'us', None, "")])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    def test_unknown_place_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None)])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    ### BULK MANY
+
+    def test_invalid_token_bulk_many(self):
+        invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
+                                             (SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
+
+    def test_valid_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
+                                                   (SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % places[0][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % places[0][1][1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+        self.assertEqual('%.3f' % places[1][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % places[1][1][1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+    def test_valid_request_components_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1),
+                                                    (SEARCH_ID_2, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_valid_request_namedplace_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_valid_request_namedplace2_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_odd_characters_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_empty_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None),
+                                                    (SEARCH_ID_2, '', None, None, None)])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
+
+    def test_empty_search_text_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '     ', 'us', None, ""),
+                                                    (SEARCH_ID_2, '     ', 'us', None, "")])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
+
+    def test_unknown_place_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None),
+                                                    (SEARCH_ID_2, '[unknown]', 'ch', None, None)])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]

server/lib/python/cartodb_services/test/test_helper.py

@@ -78,6 +78,7 @@ def plpy_mock_config():
     plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])

server/lib/python/cartodb_services/test/test_mapboxisoline.py

@@ -1,20 +1,27 @@
 import unittest
 from mock import Mock
-from cartodb_services.mapbox.isolines import MapboxIsolines
+from cartodb_services.mapbox.isolines import MapboxIsolines, DEFAULT_PROFILE
-from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
-from cartodb_services.mapbox.matrix_client import MapboxMatrixClient
 from cartodb_services.tools import Coordinate
+
 from credentials import mapbox_api_key
 
 VALID_ORIGIN = Coordinate(-73.989, 40.733)
 
 
-@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
 class MapboxIsolinesTestCase(unittest.TestCase):
 
     def setUp(self):
-        matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock())
+        self.mapbox_isolines = MapboxIsolines(apikey=mapbox_api_key(),
-        self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock())
+                                              logger=Mock())
+
+    def test_invalid_time_range(self):
+        time_ranges = [4000]
+
+        with self.assertRaises(ValueError):
+            solution = self.mapbox_isolines.calculate_isochrone(
+                origin=VALID_ORIGIN,
+                profile=DEFAULT_PROFILE,
+                time_ranges=time_ranges)
 
     def test_calculate_isochrone(self):
         time_ranges = [300, 900]

server/lib/python/cartodb_services/test/test_mapboxmatrix.py

@@ -1,58 +0,0 @@
-import unittest
-from mock import Mock
-from cartodb_services.mapbox import MapboxMatrixClient
-from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
-from cartodb_services.tools.exceptions import ServiceException
-from cartodb_services.tools import Coordinate
-from credentials import mapbox_api_key
-
-INVALID_TOKEN = 'invalid_token'
-VALID_ORIGIN = Coordinate(-73.989, 40.733)
-VALID_TARGET = Coordinate(-74, 40.733)
-VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET]
-NUM_COORDINATES_MAX = 25
-INVALID_COORDINATES_EMPTY = []
-INVALID_COORDINATES_MIN = [VALID_ORIGIN]
-INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \
-                          [VALID_TARGET
-                           for x in range(0, NUM_COORDINATES_MAX + 1)]
-VALID_PROFILE = DEFAULT_PROFILE
-INVALID_PROFILE = 'invalid_profile'
-
-
-@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199")
-class MapboxMatrixTestCase(unittest.TestCase):
-    def setUp(self):
-        self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(),
-                                                logger=Mock())
-
-    def test_invalid_profile(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(VALID_COORDINATES,
-                                      INVALID_PROFILE)
-
-    def test_invalid_coordinates_empty(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_EMPTY,
-                                      VALID_PROFILE)
-
-    def test_invalid_coordinates_max(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_MAX,
-                                      VALID_PROFILE)
-
-    def test_invalid_coordinates_min(self):
-        with self.assertRaises(ValueError):
-            self.matrix_client.matrix(INVALID_COORDINATES_MIN,
-                                      VALID_PROFILE)
-
-    def test_invalid_token(self):
-        invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock())
-        with self.assertRaises(ServiceException):
-            invalid_matrix.matrix(VALID_COORDINATES,
-                                  VALID_PROFILE)
-
-    def test_valid_request(self):
-        distance_matrix = self.matrix_client.matrix(VALID_COORDINATES,
-                                                    VALID_PROFILE)
-        assert distance_matrix