Merge branch 'master' into mapzen_geocoder_doc

This commit is contained in:
Mario de Frutos
2016-04-15 16:49:08 +02:00
20 changed files with 208 additions and 52 deletions

View File

@@ -23,11 +23,11 @@ Steps to deploy a new Data Services API version :
### Local install instructions
- install data services extension
- install data services geocoder extension
```
git clone git@github.com:CartoDB/data-services.git
data-services/geocoder/extension
cd data-services/geocoder/extension
sudo make install
```
@@ -35,7 +35,7 @@ Steps to deploy a new Data Services API version :
```
cd client && sudo make install
cd server && sudo make install
cd server/extension && sudo make install
```
- install python library
@@ -57,21 +57,28 @@ Steps to deploy a new Data Services API version :
```
# select CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
# select CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
# select CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}');
# select CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
# If sentinel is used:
SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
# If sentinel is not used
SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}');
SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}')
```
- configure plproxy to point to the same user database (you could do in a different one)
- configure plproxy to point to the a database (you can use a specific database for the server or your same user)
```
select CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}');
SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
```
- configure the search path in order to be able to execute the functions without use the schema:
- configure the search path in order to be able to execute the functions without using the schema:
```
alter role "<USER_ROLE>" set search_path='"$user", public, cartodb, cdb_dataservices_client';
ALTER ROLE "<USER_ROLE>" SET search_path="$user", public, cartodb, cdb_dataservices_client;
```

View File

@@ -208,6 +208,8 @@ UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {p
This function geocodes postal codes and country names and transforms them to points or polygon geometries. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response.
**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
Geocodes the postal code, and country name, into polygon data.
@@ -237,8 +239,6 @@ SELECT cdb_geocode_postalcode_polygon('11211', 'USA')
UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA')
```
**Note:** For the USA, US Census ZCTAs are considered.
### cdb_geocode_postalcode_point(_code text, country_name text_)
Geocodes postal codes, and country names, into point data.

View File

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \
cdb_dataservices_server--0.6.0--0.5.2.sql \
cdb_dataservices_server--0.5.2--0.6.0.sql
cdb_dataservices_server--0.6.1--0.6.0.sql \
cdb_dataservices_server--0.6.0--0.6.1.sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
TEST_DIR = test/

View File

@@ -0,0 +1,39 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,33 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
coordinates = geocoder.geocode(searchtext=searchtext, country=country)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@@ -175,6 +175,9 @@ RETURNS Geometry AS $$
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
@@ -245,6 +248,40 @@ RETURNS Geometry AS $$
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService

View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.6.0'
default_version = '0.6.1'
requires = 'plpythonu, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

View File

@@ -97,7 +97,10 @@ RETURNS Geometry AS $$
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
coordinates = geocoder.geocode(searchtext=searchtext, country=country)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])

View File

@@ -19,7 +19,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
(1 row)
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
cdb_conf_setconf
------------------

View File

@@ -11,7 +11,7 @@ CREATE EXTENSION cdb_dataservices_server;
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');

View File

@@ -17,8 +17,8 @@ class MapzenGeocoder:
self._url = base_url
@qps_retry
def geocode(self, searchtext, country=None):
request_params = self._build_requests_parameters(searchtext, country)
def geocode(self, searchtext, city=None, state_province=None, country=None):
request_params = self._build_requests_parameters(searchtext, city, state_province, country)
response = requests.get(self._url, params=request_params)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)
@@ -27,15 +27,26 @@ class MapzenGeocoder:
else:
response.raise_for_status()
def _build_requests_parameters(self, searchtext, country=None):
def _build_requests_parameters(self, searchtext, city=None,
state_province=None, country=None):
request_params = {}
request_params['text'] = searchtext
search_string = self._build_search_text(searchtext, city, state_province)
request_params['text'] = search_string
request_params['layers'] = 'address'
request_params['api_key'] = self._app_key
if country:
request_params['boundary.country'] = country
return request_params
def _build_search_text(self, searchtext, city, state_province):
search_string = searchtext
if city:
search_string = "{0}, {1}".format(search_string, city)
if state_province:
search_string = "{0}, {1}".format(search_string, state_province)
return search_string
def __parse_response(self, response):
try:
parsed_json_response = json.loads(response)

View File

@@ -1,4 +1,5 @@
import time
import random
from datetime import datetime
from exceptions import TimeoutException

View File

@@ -18,3 +18,17 @@ def polyline_to_linestring(polyline):
geometry = None
return geometry
def country_to_iso3(country):
""" Convert country to its iso3 code """
try:
country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
"'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
country_result = plpy.execute(country_plan, [country], 1)
if country_result:
return country_result[0]['iso3']
else:
return None
except BaseException as e:
plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
return None

View File

@@ -54,10 +54,10 @@ class IsolinesRoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
'period_end_date', 'username', 'orgname',
'heremaps_app_id', 'heremaps_app_code',
'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
'geocoder_type']
NOKIA_APP_ID_KEY = 'heremaps_app_id'
NOKIA_APP_CODE_KEY = 'heremaps_app_code'
NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id'
NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code'
QUOTA_KEY = 'here_isolines_quota'
SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
USERNAME_KEY = 'username'
@@ -102,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig):
self._soft_isolines_limit = True
else:
self._soft_isolines_limit = False
self._heremaps_app_id = db_config.heremaps_app_id
self._heremaps_app_code = db_config.heremaps_app_code
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
@property
def service_type(self):
@@ -168,12 +168,12 @@ class GeocoderConfig(ServiceConfig):
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date',
'heremaps_app_id', 'heremaps_app_code',
'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
'mapzen_geocoder_app_key', 'username', 'orgname']
NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
NOKIA_GEOCODER = 'heremaps'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code'
GOOGLE_GEOCODER = 'google'
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
@@ -241,8 +241,8 @@ class GeocoderConfig(ServiceConfig):
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._heremaps_app_id = db_config.heremaps_app_id
self._heremaps_app_code = db_config.heremaps_app_code
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
@@ -338,10 +338,12 @@ class ServicesDBConfig:
raise ConfigException('Here maps configuration missing')
else:
heremaps_conf = json.loads(heremaps_conf_json)
self._heremaps_app_id = heremaps_conf['app_id']
self._heremaps_app_code = heremaps_conf['app_code']
self._heremaps_geocoder_cost_per_hit = heremaps_conf[
self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
'geocoder_cost_per_hit']
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
def _get_mapzen_config(self):
mapzen_conf_json = self._get_conf('mapzen_conf')
@@ -369,12 +371,20 @@ class ServicesDBConfig:
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
@property
def heremaps_app_id(self):
return self._heremaps_app_id
def heremaps_isolines_app_id(self):
return self._heremaps_isolines_app_id
@property
def heremaps_app_code(self):
return self._heremaps_app_code
def heremaps_isolines_app_code(self):
return self._heremaps_isolines_app_code
@property
def heremaps_geocoder_app_id(self):
return self._heremaps_geocoder_app_id
@property
def heremaps_geocoder_app_code(self):
return self._heremaps_geocoder_app_code
@property
def heremaps_geocoder_cost_per_hit(self):

View File

@@ -16,21 +16,21 @@ class RedisConnection:
sentinel = Sentinel([(self._config.host,
self._config.port)],
socket_timeout=self._config.timeout)
return sentinel.master_for(
self._config.sentinel_id,
socket_timeout=self._config.timeout,
db=self._config.db
)
return sentinel.master_for(self._config.sentinel_id,
socket_timeout=self._config.timeout,
db=self._config.db,
retry_on_timeout=True)
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db)
db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
return conn
class RedisDBConfig:
DEFAULT_USER_DB = 5
DEFAULT_TIMEOUT = 2 # seconds
DEFAULT_TIMEOUT = 1.5 # seconds
def __init__(self, key, db_conn):
self._db_conn = db_conn
@@ -63,8 +63,9 @@ class RedisDBConfig:
self._sentinel_id = None
def __str__(self):
return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format(
self.host, self.port, self.sentinel_id, self.db)
return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}, " \
"Timeout: {4}".format(self.host, self.port, self.sentinel_id,
self.db, self.timeout)
@property
def host(self):

View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.4.0',
version='0.4.4',
description='CartoDB Services API Python Library',

View File

@@ -44,7 +44,7 @@ def build_plpy_mock(empty=False):
def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}]
return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":