Compare commits

..
Author SHA1 Message Date
Mario de Frutos c8ea635385 Merge pull request #141 from CartoDB/development
Version 0.6.1 sql files
2016-04-14 10:31:54 +02:00
Mario de Frutos 923774ffe4 Merge pull request #142 from CartoDB/geocoder_agnostic
Mapzen geocoder now concatenates the city and province to the query search if passed
2016-04-14 08:06:40 +02:00
Mario de Frutos cb921614c3 Version 0.6.1 sql files 2016-04-14 07:46:28 +02:00
Mario de Frutos ccd5daa63e Merge pull request #140 from CartoDB/geocoder_agnostic
Geocoder functions should be provider agnostic
2016-04-14 07:45:18 +02:00
Mario de Frutos b4c1991c90 Mapzen geocoder checks for the iso3 for the country passed 2016-04-13 18:49:13 +02:00
Carla d5c896274c Merge pull request #138 from CartoDB/docs_geocoding_extra_info
Add city geocoder information in case of coincidence
2016-04-06 10:39:06 +02:00
Carla a0377150f3 Update geocoding_functions.md 2016-04-05 14:16:47 +02:00
Carla 79e69c6020 Update geocoding_functions.md 2016-04-05 11:46:31 +02:00
Mario de Frutos c04e8fa586 Merge pull request #137 from CartoDB/development
Use retry on timeout flag for Redis connections
2016-04-04 12:00:41 +02:00
Mario de Frutos 9c47bc45e7 Merge pull request #135 from CartoDB/use_retry_on_timeout_redis
Use retry on timeout flag for Redis connections
2016-04-04 11:59:33 +02:00
Mario de Frutos a8dd211acf Use retry on timeout flag for Redis connections
In order to avoid some fails due some high load peaks in our Redis
databases, we are going to active this flag that retries 1 more time
in case it receives a socket.Timeout error while reading or connecting
2016-04-04 10:17:59 +02:00
csobier bfb8df54f4 Merge pull request #134 from CartoDB/isoline-here-link
added link to HERE coverage
2016-04-01 13:23:01 -04:00
csobier 5c23a43efa added link to HERE coverage 2016-04-01 13:09:16 -04:00
Carla ac49dc1fda Merge pull request #129 from CartoDB/iriberri-patch-1
Update README install instructions
2016-04-01 11:59:18 +02:00
Carla 3af2a8f8f0 Merge pull request #130 from CartoDB/128-independent_isoline_credentials
Make isoline credentials independent
2016-04-01 10:28:17 +02:00
Carla Iriberri 752d8317a2 Merge master 2016-04-01 10:18:46 +02:00
Carla 8c5a4d32de Update setup.py 2016-04-01 10:15:27 +02:00
Mario de Frutos 0ba513d13d Merge pull request #132 from CartoDB/development
Fixed timeout and random issues
2016-04-01 08:48:01 +02:00
Mario de Frutos f29f0769d7 Merge pull request #131 from CartoDB/fix_timeout_and_random
Fixed timeout and random problems
2016-04-01 08:46:51 +02:00
Mario de Frutos d625b26e57 Fixed timeout and random problems 2016-04-01 08:45:40 +02:00
Carla 62da0eff21 Update test_helper.py 2016-03-31 16:45:27 +02:00
Carla 5f74d012a6 Update 00_install_test files 2016-03-31 16:25:59 +02:00
Carla 42b1d6f630 Update README.md 2016-03-31 16:11:03 +02:00
Carla 7de498cc95 Add redis/sentinel examples 2016-03-31 16:01:22 +02:00
Carla Iriberri 1505e0e6ea Make isoline credentials independent 2016-03-31 15:52:52 +02:00
Carla 2668a1dfdc Update README.md 2016-03-31 15:29:18 +02:00
Carla 03ce547b8e Update README.md 2016-03-31 15:02:19 +02:00
21 changed files with 208 additions and 52 deletions
+19 -12
View File
@@ -23,11 +23,11 @@ Steps to deploy a new Data Services API version :
### Local install instructions ### Local install instructions
- install data services extension - install data services geocoder extension
``` ```
git clone git@github.com:CartoDB/data-services.git git clone git@github.com:CartoDB/data-services.git
data-services/geocoder/extension cd data-services/geocoder/extension
sudo make install sudo make install
``` ```
@@ -35,7 +35,7 @@ Steps to deploy a new Data Services API version :
``` ```
cd client && sudo make install cd client && sudo make install
cd server && sudo make install cd server/extension && sudo make install
``` ```
- install python library - install python library
@@ -57,21 +57,28 @@ Steps to deploy a new Data Services API version :
``` ```
# select CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); # If sentinel is used:
# select CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
# select CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}');
# select CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}') # If sentinel is not used
SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}');
SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}')
``` ```
- configure plproxy to point to the same user database (you could do in a different one) - configure plproxy to point to the a database (you can use a specific database for the server or your same user)
``` ```
select CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}'); SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
``` ```
- configure the search path in order to be able to execute the functions without use the schema: - configure the search path in order to be able to execute the functions without using the schema:
``` ```
alter role "<USER_ROLE>" set search_path='"$user", public, cartodb, cdb_dataservices_client'; ALTER ROLE "<USER_ROLE>" SET search_path="$user", public, cartodb, cdb_dataservices_client;
``` ```
+1 -1
View File
@@ -94,7 +94,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column},
## City geocoder ## City geocoder
The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. If there are duplicate results for a city name, the city name with the highest population will be returned.
### cdb_geocode_namedplace_point(_city_name text_) ### cdb_geocode_namedplace_point(_city_name text_)
+1 -1
View File
@@ -1,6 +1,6 @@
# Isoline Functions # Isoline Functions
The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the Here isolines service is available). The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the [HERE](https://developer.here.com/coverage-info) isolines service is available).
**This service is subject to quota limitations, and extra fees may apply**. Please view our [terms and conditions](https://cartodb.com/terms/) and check out the [Quota information section](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations related with quota usage. **This service is subject to quota limitations, and extra fees may apply**. Please view our [terms and conditions](https://cartodb.com/terms/) and check out the [Quota information section](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations related with quota usage.
+2 -2
View File
@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \ DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \ $(OLD_VERSIONS) \
cdb_dataservices_server--0.6.0--0.5.2.sql \ cdb_dataservices_server--0.6.1--0.6.0.sql \
cdb_dataservices_server--0.5.2--0.6.0.sql cdb_dataservices_server--0.6.0--0.6.1.sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
TEST_DIR = test/ TEST_DIR = test/
@@ -0,0 +1,39 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -0,0 +1,33 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
coordinates = geocoder.geocode(searchtext=searchtext, country=country)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -175,6 +175,9 @@ RETURNS Geometry AS $$
elif user_geocoder_config.google_geocoder: elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else: else:
plpy.error('Requested geocoder is not available') plpy.error('Requested geocoder is not available')
@@ -245,6 +248,40 @@ RETURNS Geometry AS $$
finally: finally:
quota_service.increment_total_service_use() quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension' comment = 'CartoDB dataservices server extension'
default_version = '0.6.0' default_version = '0.6.1'
requires = 'plpythonu, postgis, cdb_geocoder' requires = 'plpythonu, postgis, cdb_geocoder'
superuser = true superuser = true
schema = cdb_dataservices_server schema = cdb_dataservices_server
+4 -1
View File
@@ -97,7 +97,10 @@ RETURNS Geometry AS $$
try: try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key)
coordinates = geocoder.geocode(searchtext=searchtext, country=country) country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3)
if coordinates: if coordinates:
quota_service.increment_success_service_use() quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@@ -19,7 +19,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
(1 row) (1 row)
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
cdb_conf_setconf cdb_conf_setconf
------------------ ------------------
@@ -11,7 +11,7 @@ CREATE EXTENSION cdb_dataservices_server;
-- Mock the redis server connection to point to this very test db -- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
@@ -17,8 +17,8 @@ class MapzenGeocoder:
self._url = base_url self._url = base_url
@qps_retry @qps_retry
def geocode(self, searchtext, country=None): def geocode(self, searchtext, city=None, state_province=None, country=None):
request_params = self._build_requests_parameters(searchtext, country) request_params = self._build_requests_parameters(searchtext, city, state_province, country)
response = requests.get(self._url, params=request_params) response = requests.get(self._url, params=request_params)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return self.__parse_response(response.text) return self.__parse_response(response.text)
@@ -27,15 +27,26 @@ class MapzenGeocoder:
else: else:
response.raise_for_status() response.raise_for_status()
def _build_requests_parameters(self, searchtext, country=None): def _build_requests_parameters(self, searchtext, city=None,
state_province=None, country=None):
request_params = {} request_params = {}
request_params['text'] = searchtext search_string = self._build_search_text(searchtext, city, state_province)
request_params['text'] = search_string
request_params['layers'] = 'address' request_params['layers'] = 'address'
request_params['api_key'] = self._app_key request_params['api_key'] = self._app_key
if country: if country:
request_params['boundary.country'] = country request_params['boundary.country'] = country
return request_params return request_params
def _build_search_text(self, searchtext, city, state_province):
search_string = searchtext
if city:
search_string = "{0}, {1}".format(search_string, city)
if state_province:
search_string = "{0}, {1}".format(search_string, state_province)
return search_string
def __parse_response(self, response): def __parse_response(self, response):
try: try:
parsed_json_response = json.loads(response) parsed_json_response = json.loads(response)
@@ -1,4 +1,5 @@
import time import time
import random
from datetime import datetime from datetime import datetime
from exceptions import TimeoutException from exceptions import TimeoutException
@@ -18,3 +18,17 @@ def polyline_to_linestring(polyline):
geometry = None geometry = None
return geometry return geometry
def country_to_iso3(country):
""" Convert country to its iso3 code """
try:
country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
"'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
country_result = plpy.execute(country_plan, [country], 1)
if country_result:
return country_result[0]['iso3']
else:
return None
except BaseException as e:
plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
return None
@@ -54,10 +54,10 @@ class IsolinesRoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
'period_end_date', 'username', 'orgname', 'period_end_date', 'username', 'orgname',
'heremaps_app_id', 'heremaps_app_code', 'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
'geocoder_type'] 'geocoder_type']
NOKIA_APP_ID_KEY = 'heremaps_app_id' NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id'
NOKIA_APP_CODE_KEY = 'heremaps_app_code' NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code'
QUOTA_KEY = 'here_isolines_quota' QUOTA_KEY = 'here_isolines_quota'
SOFT_LIMIT_KEY = 'soft_here_isolines_limit' SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
USERNAME_KEY = 'username' USERNAME_KEY = 'username'
@@ -102,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig):
self._soft_isolines_limit = True self._soft_isolines_limit = True
else: else:
self._soft_isolines_limit = False self._soft_isolines_limit = False
self._heremaps_app_id = db_config.heremaps_app_id self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_app_code self._heremaps_app_code = db_config.heremaps_isolines_app_code
@property @property
def service_type(self): def service_type(self):
@@ -168,12 +168,12 @@ class GeocoderConfig(ServiceConfig):
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit', 'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date', 'geocoder_type', 'period_end_date',
'heremaps_app_id', 'heremaps_app_code', 'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
'mapzen_geocoder_app_key', 'username', 'orgname'] 'mapzen_geocoder_app_key', 'username', 'orgname']
NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
NOKIA_GEOCODER = 'heremaps' NOKIA_GEOCODER = 'heremaps'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code'
GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER = 'google'
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
@@ -241,8 +241,8 @@ class GeocoderConfig(ServiceConfig):
else: else:
self._soft_geocoding_limit = False self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._heremaps_app_id = db_config.heremaps_app_id self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_app_code self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
@@ -338,10 +338,12 @@ class ServicesDBConfig:
raise ConfigException('Here maps configuration missing') raise ConfigException('Here maps configuration missing')
else: else:
heremaps_conf = json.loads(heremaps_conf_json) heremaps_conf = json.loads(heremaps_conf_json)
self._heremaps_app_id = heremaps_conf['app_id'] self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
self._heremaps_app_code = heremaps_conf['app_code'] self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
self._heremaps_geocoder_cost_per_hit = heremaps_conf[ self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
'geocoder_cost_per_hit'] 'geocoder_cost_per_hit']
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
def _get_mapzen_config(self): def _get_mapzen_config(self):
mapzen_conf_json = self._get_conf('mapzen_conf') mapzen_conf_json = self._get_conf('mapzen_conf')
@@ -369,12 +371,20 @@ class ServicesDBConfig:
raise ConfigException("Malformed config for {0}: {1}".format(key, e)) raise ConfigException("Malformed config for {0}: {1}".format(key, e))
@property @property
def heremaps_app_id(self): def heremaps_isolines_app_id(self):
return self._heremaps_app_id return self._heremaps_isolines_app_id
@property @property
def heremaps_app_code(self): def heremaps_isolines_app_code(self):
return self._heremaps_app_code return self._heremaps_isolines_app_code
@property
def heremaps_geocoder_app_id(self):
return self._heremaps_geocoder_app_id
@property
def heremaps_geocoder_app_code(self):
return self._heremaps_geocoder_app_code
@property @property
def heremaps_geocoder_cost_per_hit(self): def heremaps_geocoder_cost_per_hit(self):
@@ -16,21 +16,21 @@ class RedisConnection:
sentinel = Sentinel([(self._config.host, sentinel = Sentinel([(self._config.host,
self._config.port)], self._config.port)],
socket_timeout=self._config.timeout) socket_timeout=self._config.timeout)
return sentinel.master_for( return sentinel.master_for(self._config.sentinel_id,
self._config.sentinel_id, socket_timeout=self._config.timeout,
socket_timeout=self._config.timeout, db=self._config.db,
db=self._config.db retry_on_timeout=True)
)
else: else:
conn = StrictRedis(host=self._config.host, port=self._config.port, conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db) db=self._config.db, retry_on_timeout=True,
socket_timeout=self._config.timeout)
return conn return conn
class RedisDBConfig: class RedisDBConfig:
DEFAULT_USER_DB = 5 DEFAULT_USER_DB = 5
DEFAULT_TIMEOUT = 2 # seconds DEFAULT_TIMEOUT = 1.5 # seconds
def __init__(self, key, db_conn): def __init__(self, key, db_conn):
self._db_conn = db_conn self._db_conn = db_conn
@@ -63,8 +63,9 @@ class RedisDBConfig:
self._sentinel_id = None self._sentinel_id = None
def __str__(self): def __str__(self):
return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format( return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}, " \
self.host, self.port, self.sentinel_id, self.db) "Timeout: {4}".format(self.host, self.port, self.sentinel_id,
self.db, self.timeout)
@property @property
def host(self): def host(self):
+1 -1
View File
@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup( setup(
name='cartodb_services', name='cartodb_services',
version='0.4.0', version='0.4.4',
description='CartoDB Services API Python Library', description='CartoDB Services API Python Library',
@@ -44,7 +44,7 @@ def build_plpy_mock(empty=False):
def _plpy_execute_side_effect(*args, **kwargs): def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}] return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":