Merge pull request #266 from CartoDB/development

Release 0.15.0

NEWS.md

@@ -1 +1,405 @@
+August 29, 2016
+===========
+* Released version 0.15.0 of the server
+* Geocode namedplace point functions uses Mapzen search service and in case of error
+  it'll use the internal geocoder
 
+August 19, 2016
+===========
+* Released version 0.7.4.2 of the server python library
+* Now connection errors, that intermittently come from Mapzen geocoding service, are caught and treated
+* Added more information to the logs from response
+* Fixed some errors in the QPS manager when the response object is None
+
+August 11, 2016
+===========
+* Released server version 0.14.2
+* Released client version 0.10.2
+* Always default arguments for DO functions to NULL, which prevents duplication & overwrite 
+
+August 5, 2016
+===========
+* Released server version 0.14.1
+* Fix problem with calling a logger method that doesn't exists
+
+August 3, 2016
+===========
+* Released server version 0.14
+* New logger with: plpy, rollbar integration and file
+* Added min log level to notify as config option
+* Server config to define dataservices environment for: log, third party servers, etc
+* Added logger to the SQL functions
+* Raise exception on events that should not be logged like reach the quota limit
+
+July 28, 2016
+===========
+* Release server 0.13.3.1
+* Fixed limit to 1 row for isolines with multiple range
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.13.3.1-server
+
+July 25, 2016
+===========
+* Release client 0.10.1
+* Includes an update of the `__AugmentTable` function of the client which creates an index on `cartodb_id` for the temporary table that stores the augmented results that will be afterwards joined with the original table by using this same key, `cartodb_id`.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.10.1-client
+
+July 25, 2016
+===========
+* Release server 0.13.3
+* Add provider per service
+* Default provider in case the provider is not setted
+* Refactor and improvements in the multiprovider services functions
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.13.3-server
+
+July 22, 2016
+===========
+* Release server 0.13.2
+* Fixes bug with multirange isolines #233
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.13.2-server
+
+July 15, 2016:
+===========
+* Release server 0.13.1
+* Includes a fix for the table augmentation functions in the server, which will now retrieve the client IP address and send it to the observatory functions as a new parameter. The affected functions are:
+
+  * _OBS_ConnectUserTable
+
+  * __OBS_ConnectUserTable
+
+  This change does not require any client change.
+ 
+ https://github.com/CartoDB/dataservices-api/releases/tag/0.13.1-server
+
+Jul 12, 2016:
+===========
+* Release server 0.13.0
+* [Server] Add beta augment functions, isoline fixes, observatory dump version
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.13.0-server
+
+Jul 12, 2016: 
+===========
+* Release client 0.10.0
+* [Client] Add beta augment functions, isoline fixes, observatory dump version
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-client
+
+Jul 7, 2016: 
+===========
+* Release client 0.9.0
+* This release adds two new functions in the Data Services extension client:
+  * cdb_mapzen_isodistance
+  * cdb_mapzen_isochrone
+  it also includes a bugfix for the previous release in which the explicit GRANTs to the new functions for the here, google and mapzen geocoder providers was missing in the upgrade file.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.9.0-client
+
+Jul 7, 2016: 
+===========
+* Release server 0.12.0
+* This release adds four new functions in the Data Services extension server:
+  * cdb_mapzen_isodistance
+  * cdb_mapzen_isochrone
+  * _cdb_mapzen_isolines, which contains the real isoline logic.
+  * _get_mapzen_isolines_config which retrieves the explicit configuration for the Mapzen matrix service.
+* In the Python end, this release adds the new Mapzen Matrix logic as well as the additions in the configuration and metrics file for the new service type mapzen_isolines.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.12.0-server
+
+Jul 5, 2016: 
+===========
+* Release server 0.11.0
+* Added three new public functions for each geocoding provider:
+  * cdb_here_geocode_street_point
+  * cdb_google_geocode_street_point
+  * cdb_mapzen_geocode_street_point
+* Added new function to retrieve specifically Mapzen configuration:
+  * _get_mapzen_geocoder_config
+  which uses the new class MapzenGeocoderConfig in the Python library.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.11.0-server
+
+Jul 5, 2016: 
+===========
+* Release client 0.8.0
+* Expose providers in high-precision geocoder functions
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.8.0-client
+
+Jun 15, 2016: 
+===========
+* [server python] Write quota info from services with zero padding. Closes issue #204.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server3
+
+Jun 13, 2016: 
+===========
+* [server python] Read quota info from services with and without zero padding. Closes issue #201.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server2
+
+May 31, 2016: 
+===========
+* Release client 0.7.0
+* Adds new function OBS_GetMeasureById
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.0-client
+
+May 31, 2016: 
+===========
+* Release server 0.10.0
+* Adds new function OBS_GetMeasureById
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.10.0-server
+
+May 25, 2016: 
+===========
+* Release server 0.9.0
+* Added a new routing function which allows to generate routes from an origin to a destination, which passes through a set of defined locations:
+
+  * cdb_dataservices_server.cdb_route_with_waypoints (username text, organization_name text, waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+
+  * cdb_dataservices_server._cdb_mapzen_route_with_waypoints(waypoints geometry(username text, orgname text, Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+
+  and updates the old cdb_route_point_to_point function to convert the input origin and destination geometries into an array of geometries.
+
+* Support arrays of geometries as input for the Mapzen routing Python client.
+
+* __parse_directions will now generate the locations JSON for the service from an array of geometries.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.9.0-server
+
+May 25, 2016: 
+===========
+* Release client 0.6.0
+* Includes new client function to obtain a route with waypoints:
+  * cdb_dataservices_client.cdb_route_with_waypoints (waypoints geometry(Point, 4326)[], mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0-client
+
+May 18, 2016: 
+===========
+* Release client 0.5.0
+* Added new functions for the data observatory:
+  * obs_getdemographicsnapshot(geometry);
+  * obs_getsegmentsnapshot(geometry);
+  * obs_getboundary(geometry, text);
+  * obs_getboundaryid(geometry, text);
+  * obs_getboundarybyid(text, text);
+  * obs_getboundariesbygeometry(geometry, text);
+  * obs_getboundariesbypointandradius(geometry, numeric, text);
+  * obs_getpointsbygeometry(geometry, text);
+  * obs_getpointsbypointandradius(geometry, numeric, text);
+  * obs_getmeasure(geometry, text);
+  * obs_getcategory(geometry, text);
+  * obs_getuscensusmeasure(geometry, text);
+  * obs_getuscensuscategory(geometry, text);
+  * obs_getpopulation(geometry);
+  * obs_search(text);
+  * obs_getavailableboundaries(geometry);
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.5.0-client
+
+May 18, 2016: 
+===========
+* Release server 0.8.0: Data Observatory release
+* Added new functions for the data observatory:
+  * obs_getdemographicsnapshot(geometry);
+  * obs_getsegmentsnapshot(geometry);
+  * obs_getboundary(geometry, text);
+  * obs_getboundaryid(geometry, text);
+  * obs_getboundarybyid(text, text);
+  * obs_getboundariesbygeometry(geometry, text);
+  * obs_getboundariesbypointandradius(geometry, numeric, text);
+  * obs_getpointsbygeometry(geometry, text);
+  * obs_getpointsbypointandradius(geometry, numeric, text);
+  * obs_getmeasure(geometry, text);
+  * obs_getcategory(geometry, text);
+  * obs_getuscensusmeasure(geometry, text);
+  * obs_getuscensuscategory(geometry, text);
+  * obs_getpopulation(geometry);
+  * obs_search(text);
+  * obs_getavailableboundaries(geometry);
+* Added quota manage for these new functions
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.8.0-server
+
+May 10, 2016: 
+===========
+* Release server 0.7.4
+* In case we receive a 4xx error from one of the services: isolines, here geocoder, etc we have to return an empty value an increment the empty counter. We have to raise exception in 5xx or unhandled exceptions
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.4-server
+
+May 10, 2016: 
+===========
+* Release server 0.7.3
+* Change how the blue/green system is working in the server side. Now the loopback is only in the observatory extension functions call instead in all the dataservices-api function for observatory
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.3-server
+
+May 4, 2016: 
+===========
+* Release server 0.7.2
+* Added Blue/Green capability to the data observatory functions in order to be able to use staging or production databases
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.2-server
+
+Apr 25, 2016: 
+===========
+* Release server 0.7.1
+* Use redis based config if exists, if not use the db config value
+* Refactor key to segregate more, now the services is called obs_snapshot
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.1-server
+
+Apr 21, 2016: 
+===========
+* Release client 0.4.0
+* Remove old versioning system for client side
+* Added obs_get_demography_snapshot function
+* Added obs_get_segment snapshot function
+* Integrated quota checking
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.4.0-client
+
+Apr 21, 2016: 
+===========
+* Release server 0.7.0
+* Added obs_get_demography_snapshot function
+* Added obs_get_segment snapshot function
+* Integrated quota checking
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.7.0-server
+
+Apr 19, 2016: 
+===========
+* Release server 0.6.2
+* Add Mapzen routing and geocoder quota check
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.6.2-server
+
+Apr 14, 2016: 
+===========
+* Release server 0.6.1
+* Now the implementation knows how to get the iso3 for the passed country in order to pass it to Mapzen
+* The city an the state/province parameters are used for mapzen too
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.6.1-server
+
+Apr 1, 2016:
+===========
+* Release server 0.6.0.1
+* Use specific isoline routing credentials for a provider for isoline functions, which were previously using the general credentials from the provider.
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0.1-server
+
+Mar 28, 2016:
+===========
+* Release server 0.6.0
+* Integrated Mapzen geocoder for street level geocoding function
+
+ https://github.com/CartoDB/dataservices-api/releases/tag/0.6.0-server
+ 
+Mar 23, 2016: 
+===========
+* Release server 0.5.2
+* Deleted old versioning system
+* 4xx responses returns empty routes despite to raise an exception
+* In some cases we return and empty response: one of the inputs is null, there is no generated shape for the route, etc
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.5.2-server
+
+Mar 17, 2016: 
+===========
+* Release server 0.5.1
+* Renamed the python library metrics functions
+* Create old version's folder to store the last versions
+* Refactor: Move redis and DB config logic to the python library
+* Generate the metrics log file
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.5.1-server
+
+Mar 14, 2016: 
+===========
+* Release server 0.5.0
+* Mapzen routing functions to calculate a route point to point
+* Use of Sentinel transparently
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.5.0-server
+
+Mar 14, 2016: 
+===========
+* Release client 0.3.0
+* Added cdb_routing_point_to_point function using Mapzen as provider
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.3.0-client
+
+Feb 26, 2016: 
+===========
+* Release client 0.2.0
+* Added routing isolines capabilities to the client and public API
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.2.0-client
+
+Feb 26, 2016: 
+===========
+* Release server 0.4.0
+* Added routing isolines capabilities
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.4.0-server
+
+Feb 11, 2016: 
+===========
+* Release server 0.3.0
+* Extension refactor, now is called cdb_dataservices_[client|server] in order to include more services aside the geocoder.
+* Add logic to save the metrics for the internal geocoder services as we have for the nokia and google geocoders
+* Trimmed all the inputs to avoid empty results
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.3.0-server
+
+Feb 4, 2016: 
+===========
+* Release server 0.2.0
+* Logic for the google geocoder so the users with this geocoder setted up can use street level geocoding too
+* Refactor of the python library in order to reflect the change to a services extension more than only geocoder
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.2.0-server
+
+Jan 25, 2016: 
+===========
+* Release Geocoder API 0.1.0
+* Street geocoding available through the cdb_geocoder_street_point_v2 function (only working Heremaps geocoder)
+* User config comes from Redis database
+* Increment of usage metrics for the cdb_geocoder_street_v2
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.1.0
+
+Jan 25, 2016: 
+===========
+* Release Geocoder API 0.0.1 production ready
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1
+
+Dec 23, 2015: 
+===========
+* Release Geocoder API 0.0.2 beta
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.0.2
+
+Dec 3, 2015: 
+===========
+* Release Geocoder API 0.0.1 Beta2
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta2
+
+Nov 27, 2015: 
+===========
+* Release Geocoder API BETA 1 
+  * Added the organization public user to the api key check
+
+https://github.com/CartoDB/dataservices-api/releases/tag/0.0.1-beta

README.md

@@ -77,8 +77,8 @@ Steps to deploy a new Data Services API version :
     SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
     SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
     SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}');
-    SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}')
-    SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
+    SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}');
+    SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}');
 
     # Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior)
     SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}')

client/renderer/interface.yaml

@@ -33,7 +33,6 @@
     - { name: admin1_name,  type: text}
     - { name: country_name, type: text}
 
-
 - name: cdb_geocode_postalcode_polygon
   return_type: Geometry
   params:

server/extension/cdb_dataservices_server--0.14.2--0.15.0.sql

@@ -0,0 +1,181 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.15.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_geocoder_config(text, text);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
+RETURNS boolean AS $$
+  cache_key = "user_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import GeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
+    GD[cache_key] = geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+    country_iso3 = None
+    if country_name:
+      country_iso3 = country_to_iso3(country_name)
+    coordinates = geocoder.geocode(searchtext=city_name, city=None,
+                                   state_province=admin1_name,
+                                   country=country_iso3, search_type='locality')
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode city point using mapzen')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  try:
+    if admin1_name and country_name:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
+      rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
+    elif country_name:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
+      rv = plpy.execute(plan, [city_name, country_name], 1)
+    else:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
+      rv = plpy.execute(plan, [city_name], 1)
+    result = rv[0]["mypoint"]
+    if result:
+      quota_service.increment_success_service_use()
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode namedplace point')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3, search_type='address')
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using mapzen')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu
+

server/extension/cdb_dataservices_server--0.15.0--0.14.2.sql

@@ -0,0 +1,169 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.14.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_geocoder_config(text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_geocode_namedplace(text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_internal_geocode_namedplace(text, text, text, text, text);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import GeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.tools import Logger,LoggerConfig
+
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  try:
+    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
+    rv = plpy.execute(plan, [city_name], 1)
+    result = rv[0]["mypoint"]
+    if result:
+      quota_service.increment_success_service_use()
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode namedplace point')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  try:
+    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
+    rv = plpy.execute(plan, [city_name, country_name], 1)
+    result = rv[0]["mypoint"]
+    if result:
+      quota_service.increment_success_service_use()
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode namedplace point')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  try:
+    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
+    rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
+    result = rv[0]["mypoint"]
+    if result:
+      quota_service.increment_success_service_use()
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode namedplace point')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using mapzen')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.14.2'
+default_version = '0.15.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/15_config_helper.sql

@@ -10,7 +10,7 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
 RETURNS boolean AS $$
   cache_key = "user_geocoder_config_{0}".format(username)
   if cache_key in GD:
@@ -19,7 +19,7 @@ RETURNS boolean AS $$
     from cartodb_services.metrics import GeocoderConfig
     plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
     redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname)
+    geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname, provider)
     GD[cache_key] = geocoder_config
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/20_geocode_street.sql

@@ -159,7 +159,7 @@ RETURNS Geometry AS $$
       country_iso3 = country_to_iso3(country)
     coordinates = geocoder.geocode(searchtext=searchtext, city=city,
                                    state_province=state_province,
-                                   country=country_iso3)
+                                   country=country_iso3, search_type='address')
     if coordinates:
       quota_service.increment_success_service_use()
       plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])

server/extension/sql/50_namedplaces.sql

@@ -1,76 +1,81 @@
 ---- cdb_geocode_namedplace_point(city_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
 RETURNS Geometry AS $$
-  from cartodb_services.metrics import QuotaService
-  from cartodb_services.metrics import InternalGeocoderConfig
-  from cartodb_services.tools import Logger,LoggerConfig
-
-
-  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
-
-  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
-  logger_config = GD["logger_config"]
-  logger = Logger(logger_config)
-  quota_service = QuotaService(user_geocoder_config, redis_conn)
   try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
-    rv = plpy.execute(plan, [city_name], 1)
-    result = rv[0]["mypoint"]
-    if result:
-      quota_service.increment_success_service_use()
-      return result
-    else:
-      quota_service.increment_empty_service_use()
-      return None
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
   except BaseException as e:
-    import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode namedplace point')
-  finally:
-    quota_service.increment_total_service_use()
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
 $$ LANGUAGE plpythonu;
 
 ---- cdb_geocode_namedplace_point(city_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
 RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
   from cartodb_services.metrics import QuotaService
-  from cartodb_services.metrics import InternalGeocoderConfig
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1}, {2})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname), plpy.quote_nullable('mapzen')))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
   try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
-    rv = plpy.execute(plan, [city_name, country_name], 1)
-    result = rv[0]["mypoint"]
-    if result:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
+    country_iso3 = None
+    if country_name:
+      country_iso3 = country_to_iso3(country_name)
+    coordinates = geocoder.geocode(searchtext=city_name, city=None,
+                                   state_province=admin1_name,
+                                   country=country_iso3, search_type='locality')
+    if coordinates:
       quota_service.increment_success_service_use()
-      return result
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
     else:
       quota_service.increment_empty_service_use()
       return None
   except BaseException as e:
     import sys
     quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode namedplace point')
+    logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode city point using mapzen')
   finally:
     quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
-CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.metrics import QuotaService
   from cartodb_services.metrics import InternalGeocoderConfig
@@ -86,8 +91,15 @@ RETURNS Geometry AS $$
   logger = Logger(logger_config)
   quota_service = QuotaService(user_geocoder_config, redis_conn)
   try:
-    plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
-    rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
+    if admin1_name and country_name:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
+      rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
+    elif country_name:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
+      rv = plpy.execute(plan, [city_name, country_name], 1)
+    else:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
+      rv = plpy.execute(plan, [city_name], 1)
     result = rv[0]["mypoint"]
     if result:
       quota_service.increment_success_service_use()

server/extension/test/expected/00_install_test.out

@@ -4,7 +4,6 @@ CREATE EXTENSION plpythonu;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
-CREATE EXTENSION observatory VERSION 'dev';
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;
 -- Mock the redis server connection to point to this very test db

server/extension/test/expected/50_namedplaces_test.out

@@ -27,7 +27,7 @@ INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lower
   'POINT(0.6983 39.26787)',4326)
 );
 -- Insert dummy data into country decoder table
-INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
+INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'ES');
 -- Insert dummy data into admin1 decoder table
 INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
 -- This should return the point inserted above

server/extension/test/sql/00_install_test.sql

@@ -4,7 +4,6 @@ CREATE EXTENSION plpythonu;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
 CREATE EXTENSION cdb_geocoder;
-CREATE EXTENSION observatory VERSION 'dev';
 
 -- Install the extension
 CREATE EXTENSION cdb_dataservices_server;

server/extension/test/sql/50_namedplaces_test.sql

@@ -15,7 +15,7 @@ INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lower
 );
 
 -- Insert dummy data into country decoder table
-INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
+INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'ES');
 
 -- Insert dummy data into admin1 decoder table
 INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -57,9 +57,14 @@ class HereMapsRoutingIsoline:
             return []
         else:
             self._logger.error('Error trying to calculate HERE isolines',
-                               data={"response": response.json(), "source":
-                                     source, "mode": mode, "data_range":
-                                     data_range, "range_type": range_type,
+                               data={"response_status": response.status_code,
+                                     "response_reason": response.reason,
+                                     "response_content": response.text,
+                                     "reponse_url": response.url,
+                                     "response_headers": response.headers,
+                                     "source": source, "mode": mode,
+                                     "data_range": data_range,
+                                     "range_type": range_type,
                                      "options": options})
             raise Exception('Error trying to calculate HERE isolines')
 

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -18,23 +18,45 @@ class MapzenGeocoder:
         self._logger = logger
 
     @qps_retry
-    def geocode(self, searchtext, city=None, state_province=None, country=None):
-        request_params = self._build_requests_parameters(searchtext, city, state_province, country)
-        response = requests.get(self._url, params=request_params)
-        if response.status_code == requests.codes.ok:
-            return self.__parse_response(response.text)
-        elif response.status_code == requests.codes.bad_request:
+    def geocode(self, searchtext, city=None, state_province=None, country=None, search_type=None):
+        request_params = self._build_requests_parameters(searchtext, city,
+                                                         state_province,
+                                                         country, search_type)
+        try:
+            response = requests.get(self._url, params=request_params)
+            if response.status_code == requests.codes.ok:
+                return self.__parse_response(response.text)
+            elif response.status_code == requests.codes.bad_request:
+                return []
+            else:
+                self._logger.error('Error trying to geocode using mapzen',
+                                data={"response_status": response.status_code,
+                                        "response_reason": response.reason,
+                                        "response_content": response.text,
+                                        "reponse_url": response.url,
+                                        "response_headers": response.headers,
+                                        "searchtext": searchtext,
+                                        "city": city, "country": country,
+                                        "state_province": state_province })
+                raise Exception('Error trying to geocode {0} using mapzen'.format(searchtext))
+        except requests.ConnectionError as e:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to Mapzen geocoding server',
+                               exception=e)
             return []
-        else:
-            raise Exception('Error trying to geocode {0} using mapzen'.format(searchtext))
+
 
     def _build_requests_parameters(self, searchtext, city=None,
-                                   state_province=None, country=None):
+                                   state_province=None, country=None,
+                                   search_type=None):
         request_params = {}
-        search_string = self._build_search_text(searchtext, city, state_province)
+        search_string = self._build_search_text(searchtext.strip(),
+                                                city,
+                                                state_province)
         request_params['text'] = search_string
-        request_params['layers'] = 'address'
         request_params['api_key'] = self._app_key
+        if search_type:
+            request_params['layers'] = search_type
         if country:
             request_params['boundary.country'] = country
         return request_params

server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py

@@ -44,8 +44,13 @@ class MatrixClient:
 
         if not requests.codes.ok:
             self._logger.error('Error trying to get matrix distance from mapzen',
-                               data={"response": response, "locations":
-                                     locations, "costing": costing})
+                               data={"response_status": response.status_code,
+                                     "response_reason": response.reason,
+                                     "response_content": response.text,
+                                     "reponse_url": response.url,
+                                     "response_headers": response.headers,
+                                     "locations": locations,
+                                     "costing": costing})
             raise Exception('Error trying to get matrix distance from mapzen')
 
         return response.json()

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -26,7 +26,8 @@ class QPSService:
             try:
                 return fn(*args, **kwargs)
             except Exception as e:
-                if hasattr(e, 'response') and (e.response.status_code == 429):
+                response = getattr(e, 'response', None)
+                if response and (response.status_code == 429):
                     self.retry(start_time, attempt_number)
                 else:
                     raise e

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -49,11 +49,15 @@ class MapzenRouting:
         elif response.status_code == requests.codes.bad_request:
             return MapzenRoutingResponse(None, None, None)
         else:
-            self._logger.error('Error trying to calculate route using HERE',
-                               data={"response": response.json(), "waypoints":
-                                     waypoints, "mode": mode, "options":
-                                     options})
-            raise Exception('Error trying to calculate route using HERE')
+            self._logger.error('Error trying to calculate route using Mapzen',
+                               data={"response_status": response.status_code,
+                                     "response_reason": response.reason,
+                                     "response_content": response.text,
+                                     "reponse_url": response.url,
+                                     "response_headers": response.headers,
+                                     "waypoints": waypoints, "mode": mode,
+                                     "options": options})
+            raise Exception('Error trying to calculate route using Mapzen')
 
     def __parse_options(self, options):
         return dict(option.split('=') for option in options)

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -286,11 +286,11 @@ class GeocoderConfig(ServiceConfig):
     PERIOD_END_DATE = 'period_end_date'
     DEFAULT_PROVIDER = 'mapzen'
 
-    def __init__(self, redis_connection, db_conn, username, orgname=None):
+    def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
         super(GeocoderConfig, self).__init__(redis_connection, db_conn,
                                              username, orgname)
         filtered_config = {key: self._redis_config[key] for key in self.GEOCODER_CONFIG_KEYS if key in self._redis_config.keys()}
-        self.__parse_config(filtered_config, self._db_config)
+        self.__parse_config(filtered_config, self._db_config, forced_provider)
         self.__check_config(filtered_config)
 
     def __check_config(self, filtered_config):
@@ -307,9 +307,12 @@ class GeocoderConfig(ServiceConfig):
 
         return True
 
-    def __parse_config(self, filtered_config, db_config):
-        self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
-        if not self._geocoder_provider:
+    def __parse_config(self, filtered_config, db_config, forced_provider):
+        if forced_provider:
+            self._geocoder_provider = forced_provider
+        elif filtered_config[self.GEOCODER_PROVIDER].lower():
+            self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
+        else:
             self._geocoder_provider = self.DEFAULT_PROVIDER
         self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.7.4.1',
+    version='0.8',
 
     description='CartoDB Services API Python Library',