Exposes geocoder providers in public geocoder functions

Add config function in postgres explicitly to get MapzenGeocoderConfig.
Bump versions for client and server APIs. New MapzenGeocoderConfig
included to be able to use current QuotaServices with non-configured
users.

client/Makefile

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
 DATA =  $(NEW_EXTENSION_ARTIFACT) \
   $(OLD_VERSIONS) \
-  cdb_dataservices_client--0.6.0--0.7.0.sql \
-  cdb_dataservices_client--0.7.0--0.6.0.sql
+  cdb_dataservices_client--0.7.0--0.8.0.sql \
+  cdb_dataservices_client--0.8.0--0.7.0.sql
 
 
 REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))

client/cdb_dataservices_client--0.7.0--0.8.0.sql

@@ -0,0 +1,113 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.8.0'" to load this file. \quit
+
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+DECLARE
+  ret Geometry;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
+    RETURN ret;
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+DECLARE
+  ret Geometry;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
+    RETURN ret;
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+DECLARE
+  ret Geometry;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
+    RETURN ret;
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
+  
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
+  
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
+  
+$$ LANGUAGE plproxy;

client/cdb_dataservices_client--0.8.0--0.7.0.sql

@@ -0,0 +1,9 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.7.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_here_geocode_street_point (text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_google_geocode_street_point (text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_geocode_street_point (text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (text, text, text, text, text, text);

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.7.0'
+default_version = '0.8.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/renderer/interface.yaml

@@ -59,6 +59,30 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: cdb_here_geocode_street_point
+  return_type: Geometry
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
+- name: cdb_google_geocode_street_point
+  return_type: Geometry
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
+- name: cdb_mapzen_geocode_street_point
+  return_type: Geometry
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
 - name: cdb_isodistance
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true

server/extension/cdb_dataservices_server--0.10.0--0.11.0.sql

@@ -0,0 +1,101 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.11.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  if user_geocoder_config.heremaps_geocoder:
+    here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    plpy.error('Here geocoder is not available for your account.')
+
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  if user_geocoder_config.google_geocoder:
+    google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+    return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  else:
+    plpy.error('Google geocoder is not available for your account.')
+
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
+
+  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_mapzen_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import MapzenGeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = mapzen_geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/cdb_dataservices_server--0.11.0--0.10.0.sql

@@ -0,0 +1,46 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.10.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3)
+    if coordinates:
+      quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
+    plpy.notice(traceback.format_tb(traceback_))
+    plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.10.0'
+default_version = '0.11.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/15_config_helper.sql

@@ -12,6 +12,20 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_mapzen_geocoder_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import MapzenGeocoderConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = mapzen_geocoder_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_internal_geocoder_config_{0}".format(username)

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,3 +1,3 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
+from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -128,6 +128,60 @@ class RoutingConfig(ServiceConfig):
     def period_end_date(self):
         return self._period_end_date
 
+# Explicit class for the geocoder configuration for Mapzen
+# which does not require users to be configured to use the service
+class MapzenGeocoderConfig(ServiceConfig):
+
+    PERIOD_END_DATE = 'period_end_date'
+
+    def __init__(self, redis_connection, db_conn, username, orgname=None):
+        super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn,
+                                            username, orgname)
+        self._log_path = self._db_config.geocoder_log_path
+        try:
+            self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key
+            self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota
+            self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
+            self._cost_per_hit = 0
+        except Exception as e:
+            raise ConfigException("Malformed config for Mapzen geocoder: {1}".format(key, e))
+
+    @property
+    def service_type(self):
+        return 'geocoder_mapzen'
+
+    @property
+    def mapzen_api_key(self):
+        return self._mapzen_api_key
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def google_geocoder(self):
+        return None
+
+    @property
+    def geocoding_quota(self):
+        return self._monthly_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return False
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def log_path(self):
+        return self._log_path
+
 
 class IsolinesRoutingConfig(ServiceConfig):
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.6.4',
+    version='0.6.5',
 
     description='CartoDB Services API Python Library',