Merge pull request #141 from CartoDB/development

Version 0.6.1 sql files

README.md

@@ -23,11 +23,11 @@ Steps to deploy a new Data Services API version :
 
 ### Local install instructions
 
-- install data services extension 
+- install data services geocoder extension 
 
     ```
     git clone git@github.com:CartoDB/data-services.git
-    data-services/geocoder/extension
+    cd data-services/geocoder/extension
     sudo make install
     ```
 
@@ -35,7 +35,7 @@ Steps to deploy a new Data Services API version :
     
     ```
     cd client && sudo make install
-    cd server && sudo make install
+    cd server/extension && sudo make install
     ```
 
 - install python library
@@ -57,21 +57,28 @@ Steps to deploy a new Data Services API version :
 
 
     ```
-    # select CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
-    # select CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
-    
-    # select CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}');
-    # select CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
+    # If sentinel is used:
+      SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
+      SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
+      
+    # If sentinel is not used
+      SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
+      SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
+  
+    SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}');
+    SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
+    SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}');
+    SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}')
     ```
 
-- configure plproxy to point to the same user database (you could do in a different one)
+- configure plproxy to point to the a database (you can use a specific database for the server or your same user)
 
     ```
-     select CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}');
+     SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
     ```
 
-- configure the search path in order to be able to execute the functions without use the schema:
+- configure the search path in order to be able to execute the functions without using the schema:
 
     ```
-    alter role "<USER_ROLE>" set search_path='"$user", public, cartodb, cdb_dataservices_client';
+    ALTER ROLE "<USER_ROLE>" SET search_path="$user", public, cartodb, cdb_dataservices_client;
     ```

doc/API.md

@@ -9,6 +9,7 @@ The isoline functions provide a way to generate isolines in terms of distance an
 ## Documentation
 
 * [Quickstart](quickstart.md)
-* [General concepts](general_concepts.md)
-* [Geocoding functions](geocoding_functions.md)
-* [Isoline functions](isoline_functions.md)
+* [General Concepts](general_concepts.md)
+* [Geocoding Functions](geocoding_functions.md)
+* [Isoline Functions](isoline_functions.md)
+* [Quota Information](quota_information.md)

doc/general_concepts.md

@@ -1,4 +1,4 @@
-# General concepts
+# General Concepts
 
 The Data Services API offers geocoding and isoline services on top of the CartoDB SQL API by means of a set of functions. Each one of these functions is oriented to one kind of operation and returns the corresponding geometry (a `polygon` or a `point`), according to the input information.
 

doc/geocoding_functions.md

@@ -1,4 +1,4 @@
-# Geocoding functions
+# Geocoding Functions
 
 The following geocoding functions are available, grouped by categories.
 
@@ -94,7 +94,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column},
 
 ## City geocoder
 
-The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name.
+The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. If there are duplicate results for a city name, the city name with the highest population will be returned.
 
 ### cdb_geocode_namedplace_point(_city_name text_)
 
@@ -181,6 +181,8 @@ UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {p
 
 The following functions provide a postal code geocoding service that can be used to obtain points or polygon results. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response.
 
+**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
+
 ### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
 
 #### Arguments
@@ -208,8 +210,6 @@ SELECT cdb_geocode_postalcode_polygon('11211', 'USA')
 UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA')
 ```
 
-**Note:** For the USA, US Census ZCTAs are considered.
-
 ### cdb_geocode_postalcode_point(_code text, country_name text_)
 
 #### Arguments

doc/isoline_functions.md

@@ -1,6 +1,6 @@
-# Isoline functions
+# Isoline Functions
 
-The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the Here isolines service is available).
+The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the [HERE](https://developer.here.com/coverage-info) isolines service is available).
 
 **This service is subject to quota limitations, and extra fees may apply**. Please view our [terms and conditions](https://cartodb.com/terms/) and check out the [Quota information section](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations related with quota usage.
 

doc/quota_information.md

@@ -1,4 +1,4 @@
-# Quota information
+# Quota Information
 
 **This Data Services API provides functions which are subject to quota limitations, and extra fees may apply**. Please check our [terms and conditions](https://cartodb.com/terms/).
 

server/extension/Makefile

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
 DATA =  $(NEW_EXTENSION_ARTIFACT) \
   $(OLD_VERSIONS) \
-  cdb_dataservices_server--0.5.2--0.5.1.sql \
-  cdb_dataservices_server--0.5.1--0.5.2.sql
+  cdb_dataservices_server--0.6.1--0.6.0.sql \
+  cdb_dataservices_server--0.6.0--0.6.1.sql
 
 REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
 TEST_DIR = test/

server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql

@@ -0,0 +1,39 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql

@@ -0,0 +1,33 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
+    coordinates = geocoder.geocode(searchtext=searchtext, country=country)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.5.2'
+default_version = '0.6.1'
 requires = 'plpythonu, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.5.2--0.6.0.sql

@@ -0,0 +1,54 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  if user_geocoder_config.heremaps_geocoder:
+    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif user_geocoder_config.google_geocoder:
+    google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif user_geocoder_config.mapzen_geocoder:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    plpy.error('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
+    coordinates = geocoder.geocode(searchtext, country=country)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as point; ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['point']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/old_versions/cdb_dataservices_server--0.5.2.sql

@@ -31,11 +31,6 @@ RETURNS cdb_dataservices_server.simple_route AS $$
   try:
     client = MapzenRouting(user_routing_config.mapzen_app_key)
 
-    if not origin or not destination:
-      plpy.notice("Empty origin or destination")
-      quota_service.increment_empty_service_use()
-      return [None, None, None]
-
     orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat']
     orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon']
     origin_coordinates = Coordinate(orig_lon, orig_lat)
@@ -47,21 +42,20 @@ RETURNS cdb_dataservices_server.simple_route AS $$
 
     if resp and resp.shape:
       shape_linestring = polyline_to_linestring(resp.shape)
-      if shape_linestring:
-        quota_service.increment_success_service_use()
-        return [shape_linestring, resp.length, resp.duration]
-      else:
-        quota_service.increment_empty_service_use()
-        return [None, None, None]
+      quota_service.increment_success_service_use()
+      return [shape_linestring, resp.length, resp.duration]
     else:
       quota_service.increment_empty_service_use()
-      return [None, None, None]
   except BaseException as e:
     import sys, traceback
     type_, value_, traceback_ = sys.exc_info()
     quota_service.increment_failed_service_use()
     error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e)
-    plpy.notice(traceback.format_tb(traceback_))
+    plpy.warning('Resp shape: {0}'.format(resp.shape))
+    plpy.warning('Resp duration: {0}'.format(resp.duration))
+    plpy.warning('Resp length: {0}'.format(resp.length))
+    plpy.warning('Points: {0}, {1}'.format(origin, destination))
+    plpy.warning(traceback.format_tb(traceback_))
     plpy.error(error_msg)
   finally:
     quota_service.increment_total_service_use()

server/extension/old_versions/cdb_dataservices_server--0.6.0--0.5.2.sql

@@ -0,0 +1,22 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.5.2'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  if user_geocoder_config.heremaps_geocoder:
+    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif user_geocoder_config.google_geocoder:
+    google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    plpy.error('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu;

server/extension/sql/20_geocode_street.sql

@@ -12,6 +12,9 @@ RETURNS Geometry AS $$
   elif user_geocoder_config.google_geocoder:
     google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
     return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  elif user_geocoder_config.mapzen_geocoder:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
   else:
     plpy.error('Requested geocoder is not available')
 
@@ -82,3 +85,37 @@ RETURNS Geometry AS $$
   finally:
     quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/test/expected/00_install_test.out

@@ -19,13 +19,13 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
+SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}');
  cdb_conf_setconf 
 ------------------
  

server/extension/test/sql/00_install_test.sql

@@ -11,8 +11,8 @@ CREATE EXTENSION cdb_dataservices_server;
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
 SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
-SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
-SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}');
+SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 
 -- Mock the varnish invalidation function

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -1 +1,2 @@
 from routing import MapzenRouting, MapzenRoutingResponse
+from geocoder import MapzenGeocoder

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -0,0 +1,64 @@
+import requests
+import json
+import re
+
+from exceptions import WrongParams, MalformedResult
+from qps import qps_retry
+from cartodb_services.tools import Coordinate, PolyLine
+
+
+class MapzenGeocoder:
+    'A Mapzen Geocoder wrapper for python'
+
+    BASE_URL = 'https://search.mapzen.com/v1/search'
+
+    def __init__(self, app_key, base_url=BASE_URL):
+        self._app_key = app_key
+        self._url = base_url
+
+    @qps_retry
+    def geocode(self, searchtext, city=None, state_province=None, country=None):
+        request_params = self._build_requests_parameters(searchtext, city, state_province, country)
+        response = requests.get(self._url, params=request_params)
+        if response.status_code == requests.codes.ok:
+            return self.__parse_response(response.text)
+        elif response.status_code == requests.codes.bad_request:
+            return []
+        else:
+            response.raise_for_status()
+
+    def _build_requests_parameters(self, searchtext, city=None,
+                                   state_province=None, country=None):
+        request_params = {}
+        search_string = self._build_search_text(searchtext, city, state_province)
+        request_params['text'] = search_string
+        request_params['layers'] = 'address'
+        request_params['api_key'] = self._app_key
+        if country:
+            request_params['boundary.country'] = country
+        return request_params
+
+    def _build_search_text(self, searchtext, city, state_province):
+        search_string = searchtext
+        if city:
+            search_string = "{0}, {1}".format(search_string, city)
+        if state_province:
+            search_string = "{0}, {1}".format(search_string, state_province)
+
+        return search_string
+
+    def __parse_response(self, response):
+        try:
+            parsed_json_response = json.loads(response)
+            feature = parsed_json_response['features'][0]
+            return self._extract_lng_lat_from_result(feature)
+        except IndexError:
+            return []
+        except KeyError:
+            raise MalformedResult()
+
+    def _extract_lng_lat_from_result(self, result):
+        location = result['geometry']['coordinates']
+        longitude = location[0]
+        latitude = location[1]
+        return [longitude, latitude]

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -1,4 +1,5 @@
 import time
+import random
 from datetime import datetime
 from exceptions import TimeoutException
 

server/lib/python/cartodb_services/cartodb_services/mapzen/types.py

@@ -18,3 +18,17 @@ def polyline_to_linestring(polyline):
         geometry = None
 
     return geometry
+
+def country_to_iso3(country):
+    """ Convert country to its iso3 code """
+    try:
+        country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
+                                    "'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
+        country_result = plpy.execute(country_plan, [country], 1)
+        if country_result:
+            return country_result[0]['iso3']
+        else:
+            return None
+    except BaseException as e:
+        plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
+        return None

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -54,10 +54,10 @@ class IsolinesRoutingConfig(ServiceConfig):
 
     ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
                            'period_end_date', 'username', 'orgname',
-                           'heremaps_app_id', 'heremaps_app_code',
+                           'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
                            'geocoder_type']
-    NOKIA_APP_ID_KEY = 'heremaps_app_id'
-    NOKIA_APP_CODE_KEY = 'heremaps_app_code'
+    NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id'
+    NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code'
     QUOTA_KEY = 'here_isolines_quota'
     SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
     USERNAME_KEY = 'username'
@@ -102,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig):
             self._soft_isolines_limit = True
         else:
             self._soft_isolines_limit = False
-        self._heremaps_app_id = db_config.heremaps_app_id
-        self._heremaps_app_code = db_config.heremaps_app_code
+        self._heremaps_app_id = db_config.heremaps_isolines_app_id
+        self._heremaps_app_code = db_config.heremaps_isolines_app_code
 
     @property
     def service_type(self):
@@ -168,15 +168,17 @@ class GeocoderConfig(ServiceConfig):
     GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
                             'geocoding_quota', 'soft_geocoding_limit',
                             'geocoder_type', 'period_end_date',
-                            'heremaps_app_id', 'heremaps_app_code', 'username',
-                            'orgname']
-    NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
+                            'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
+                            'mapzen_geocoder_app_key', 'username', 'orgname']
+    NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
     NOKIA_GEOCODER = 'heremaps'
-    NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
-    NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
+    NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id'
+    NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code'
     GOOGLE_GEOCODER = 'google'
     GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
     GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
+    MAPZEN_GEOCODER = 'mapzen'
+    MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_app_key'
     GEOCODER_TYPE = 'geocoder_type'
     QUOTA_KEY = 'geocoding_quota'
     SOFT_LIMIT_KEY = 'soft_geocoding_limit'
@@ -217,11 +219,15 @@ class GeocoderConfig(ServiceConfig):
 
     def __check_config(self, filtered_config):
         if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
-            if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
+            if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
+            not self.heremaps_app_id or not self.heremaps_app_code:
                 raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
         elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
             if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
                 raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
+        elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
+            if not self.mapzen_app_key:
+                raise ConfigException("""Mapzen config is not setted up""")
 
         return True
 
@@ -235,18 +241,23 @@ class GeocoderConfig(ServiceConfig):
         else:
             self._soft_geocoding_limit = False
         if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
-            self._heremaps_app_id = db_config.heremaps_app_id
-            self._heremaps_app_code = db_config.heremaps_app_code
+            self._heremaps_app_id = db_config.heremaps_geocoder_app_id
+            self._heremaps_app_code = db_config.heremaps_geocoder_app_code
             self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
         elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
             self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
             self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
             self._cost_per_hit = 0
+        elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
+            self._mapzen_app_key = db_config.mapzen_geocoder_app_key
+            self._cost_per_hit = 0
 
     @property
     def service_type(self):
         if self._geocoder_type == self.GOOGLE_GEOCODER:
             return 'geocoder_google'
+        elif self._geocoder_type == self.MAPZEN_GEOCODER:
+            return 'geocoder_mapzen'
         else:
             return 'geocoder_here'
 
@@ -258,6 +269,10 @@ class GeocoderConfig(ServiceConfig):
     def google_geocoder(self):
         return self._geocoder_type == self.GOOGLE_GEOCODER
 
+    @property
+    def mapzen_geocoder(self):
+        return self._geocoder_type == self.MAPZEN_GEOCODER
+
     @property
     def google_client_id(self):
         return self._google_maps_client_id
@@ -289,6 +304,10 @@ class GeocoderConfig(ServiceConfig):
     def heremaps_app_code(self):
         return self._heremaps_app_code
 
+    @property
+    def mapzen_app_key(self):
+        return self._mapzen_app_key
+
     @property
     def is_high_resolution(self):
         return True
@@ -319,10 +338,12 @@ class ServicesDBConfig:
             raise ConfigException('Here maps configuration missing')
         else:
             heremaps_conf = json.loads(heremaps_conf_json)
-            self._heremaps_app_id = heremaps_conf['app_id']
-            self._heremaps_app_code = heremaps_conf['app_code']
-            self._heremaps_geocoder_cost_per_hit = heremaps_conf[
+            self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
+            self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
+            self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
                 'geocoder_cost_per_hit']
+            self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
+            self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
 
     def _get_mapzen_config(self):
         mapzen_conf_json = self._get_conf('mapzen_conf')
@@ -331,6 +352,7 @@ class ServicesDBConfig:
         else:
             mapzen_conf = json.loads(mapzen_conf_json)
             self._mapzen_routing_app_key = mapzen_conf['routing_app_key']
+            self._mapzen_geocoder_app_key = mapzen_conf['geocoder_app_key']
 
     def _get_logger_config(self):
         logger_conf_json = self._get_conf('logger_conf')
@@ -349,12 +371,20 @@ class ServicesDBConfig:
             raise ConfigException("Malformed config for {0}: {1}".format(key, e))
 
     @property
-    def heremaps_app_id(self):
-        return self._heremaps_app_id
+    def heremaps_isolines_app_id(self):
+        return self._heremaps_isolines_app_id
 
     @property
-    def heremaps_app_code(self):
-        return self._heremaps_app_code
+    def heremaps_isolines_app_code(self):
+        return self._heremaps_isolines_app_code
+
+    @property
+    def heremaps_geocoder_app_id(self):
+        return self._heremaps_geocoder_app_id
+
+    @property
+    def heremaps_geocoder_app_code(self):
+        return self._heremaps_geocoder_app_code
 
     @property
     def heremaps_geocoder_cost_per_hit(self):
@@ -364,6 +394,10 @@ class ServicesDBConfig:
     def mapzen_routing_app_key(self):
         return self._mapzen_routing_app_key
 
+    @property
+    def mapzen_geocoder_app_key(self):
+        return self._mapzen_geocoder_app_key
+
     @property
     def geocoder_log_path(self):
         return self._geocoder_log_path

server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py

@@ -16,21 +16,21 @@ class RedisConnection:
             sentinel = Sentinel([(self._config.host,
                                   self._config.port)],
                                 socket_timeout=self._config.timeout)
-            return sentinel.master_for(
-                self._config.sentinel_id,
-                socket_timeout=self._config.timeout,
-                db=self._config.db
-            )
+            return sentinel.master_for(self._config.sentinel_id,
+                                       socket_timeout=self._config.timeout,
+                                       db=self._config.db,
+                                       retry_on_timeout=True)
         else:
             conn = StrictRedis(host=self._config.host, port=self._config.port,
-                               db=self._config.db)
+                               db=self._config.db, retry_on_timeout=True,
+                               socket_timeout=self._config.timeout)
             return conn
 
 
 class RedisDBConfig:
 
     DEFAULT_USER_DB = 5
-    DEFAULT_TIMEOUT = 2  # seconds
+    DEFAULT_TIMEOUT = 1.5  # seconds
 
     def __init__(self, key, db_conn):
         self._db_conn = db_conn
@@ -63,8 +63,9 @@ class RedisDBConfig:
                 self._sentinel_id = None
 
     def __str__(self):
-        return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format(
-            self.host, self.port, self.sentinel_id, self.db)
+        return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}, " \
+               "Timeout: {4}".format(self.host, self.port, self.sentinel_id,
+                                     self.db, self.timeout)
 
     @property
     def host(self):

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.3.3',
+    version='0.4.4',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_helper.py

@@ -44,8 +44,8 @@ def build_plpy_mock(empty=False):
 
 def _plpy_execute_side_effect(*args, **kwargs):
     if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
-        return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}]
+        return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
-        return [{'conf': '{"routing_app_key": "app_key"}'}]
+        return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
         return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]

server/lib/python/cartodb_services/test/test_mapzengeocoder.py

@@ -0,0 +1,109 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+import unittest
+import requests_mock
+
+from cartodb_services.mapzen import MapzenGeocoder
+from cartodb_services.mapzen.exceptions import MalformedResult
+
+requests_mock.Mocker.TEST_PREFIX = 'test_'
+
+
+@requests_mock.Mocker()
+class GoogleGeocoderTestCase(unittest.TestCase):
+    MAPZEN_GEOCODER_URL = 'https://search.mapzen.com/v1/search'
+
+    EMPTY_RESPONSE = """{
+       "results" : [],
+       "status" : "ZERO_RESULTS"
+    }"""
+
+    GOOD_RESPONSE = """{
+      "geocoding": {
+        "version": "0.1",
+        "attribution": "https://search.mapzen.com/v1/attribution",
+        "query": {
+          "text": "Calle siempreviva 3, Valladolid",
+          "parsed_text": {
+            "name": "Calle siempreviva 3",
+            "regions": [
+              "Calle siempreviva 3",
+              "Valladolid"
+            ],
+            "admin_parts": "Valladolid"
+          },
+          "types": {
+            "from_layers": [
+              "osmaddress",
+              "openaddresses"
+            ]
+          },
+          "size": 10,
+          "private": false,
+          "type": [
+            "osmaddress",
+            "openaddresses"
+          ],
+          "querySize": 20
+        },
+        "engine": {
+          "name": "Pelias",
+          "author": "Mapzen",
+          "version": "1.0"
+        },
+        "timestamp": 1458661873749
+      },
+      "type": "FeatureCollection",
+      "features": [
+        {
+          "type": "Feature",
+          "properties": {
+            "id": "df7428b955ae44a39dc40d52578f61e3",
+            "gid": "oa:address:df7428b955ae44a39dc40d52578f61e3",
+            "layer": "address",
+            "source": "oa",
+            "name": "5 Close Siempreviva",
+            "housenumber": "5",
+            "street": "Close Siempreviva",
+            "country_a": "ESP",
+            "country": "Spain",
+            "region": "Valladolid",
+            "localadmin": "Valladolid",
+            "locality": "Valladolid",
+            "confidence": 0.887,
+            "label": "5 Close Siempreviva, Valladolid, Spain"
+          },
+          "geometry": {
+            "type": "Point",
+            "coordinates": [
+              -4.730928,
+              41.669034
+            ]
+          }
+        }
+      ]
+    }"""
+
+    MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
+
+    def setUp(self):
+        self.geocoder = MapzenGeocoder('search-XXXXXXX')
+
+    def test_geocode_address_with_valid_params(self, req_mock):
+        req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL,
+                       text=self.GOOD_RESPONSE)
+        response = self.geocoder.geocode(
+            searchtext='Calle Siempreviva 3, Valldolid',
+            country='ESP')
+
+        self.assertEqual(response[0], -4.730928)
+        self.assertEqual(response[1], 41.669034)
+
+    def test_geocode_with_malformed_result(self, req_mock):
+        req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL,
+                       text=self.MALFORMED_RESPONSE)
+        with self.assertRaises(MalformedResult):
+            self.geocoder.geocode(
+                searchtext='Calle Siempreviva 3, Valladolid',
+                country='ESP')

server/lib/python/cartodb_services/test/test_user_service.py

@@ -87,5 +87,5 @@ class TestUserService(TestCase):
                                                quota=quota, end_date=end_date)
         plpy_mock = test_helper.build_plpy_mock()
         geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
-                                         username, orgname,)
+                                         username, orgname)
         return UserMetricsService(geocoder_config, self.redis_conn)