Compare commits

...

22 Commits

Author SHA1 Message Date
Mario de Frutos
efde2ee1b7 Fix typo in the default value for service providers 2016-07-25 14:00:29 +02:00
Mario de Frutos
fc399c12c1 Change notices for debug 2016-07-25 13:58:01 +02:00
Mario de Frutos
f03897257e Better than lower, change to empty string if not provider setting in redis 2016-07-25 13:40:06 +02:00
Mario de Frutos
90998d4819 Fix users without provider setting setted 2016-07-25 13:35:59 +02:00
Mario de Frutos
5c5bed5557 Merge pull request #236 from CartoDB/development
Release 0.13.3
2016-07-25 13:26:27 +02:00
Mario de Frutos
d0f8797ccb Release 0.13.3 complete artifact 2016-07-25 13:25:36 +02:00
Mario de Frutos
eb2c713b0e Merge pull request #231 from CartoDB/user_service_providers
Add service providers that come from the user configuration
2016-07-25 13:19:44 +02:00
Mario de Frutos
79d0b5ba7c Add service providers that come from the user configuration 2016-07-25 12:25:52 +02:00
Mario de Frutos
15eb8633d7 Merge pull request #234 from CartoDB/development
Release 0.13.2
2016-07-22 16:30:35 +02:00
Mario de Frutos
ea4ee402a8 Release 0.13.2 to fix range isolines problem 2016-07-22 16:27:44 +02:00
Mario de Frutos
e8d55ec68a Merge pull request #232 from CartoDB/fix_isolines_range_bug
Fix isolines range bug
2016-07-22 16:24:48 +02:00
Mario de Frutos
69548eaa25 Fixed return error in case NULL value is returned for isolines shape 2016-07-22 14:55:01 +02:00
Mario de Frutos
ff8eed9750 Fix empty component return in isolines 2016-07-22 14:54:53 +02:00
Mario de Frutos
91c715f9f5 Update README.md 2016-07-20 18:14:41 +02:00
Carla
d5b6d1f5f1 Merge pull request #227 from CartoDB/development
Release server 0.13.1
2016-07-15 18:12:39 +02:00
Carla
28f49bdad2 Merge pull request #226 from CartoDB/release_server_13_1
Release server 0.13.1
2016-07-15 15:33:30 +02:00
Carla
3c63c4ec43 Merge pull request #225 from CartoDB/fix_client_addr_augment
Fix client address recognition in augment functions
2016-07-15 15:31:16 +02:00
Carla Iriberri
8e4c1554bb Release 0.13.1 server 2016-07-15 15:19:10 +02:00
Carla Iriberri
39a7cc955a Move IP address reading to DS server 2016-07-15 11:54:38 +02:00
Carla
1ee7992504 fix typo goecodes -> geocodes 2016-07-12 17:00:55 +02:00
Carla
6cd53b519b Merge pull request #224 from CartoDB/development
Release Client 0.10.0, Server 0.13.0
2016-07-12 14:06:40 +02:00
Carla
64e02e2f77 Merge pull request #218 from CartoDB/development
Release client 0.9.0 and server 0.12.0
2016-07-07 11:38:19 +02:00
29 changed files with 7688 additions and 211 deletions

View File

@@ -76,7 +76,7 @@ Steps to deploy a new Data Services API version :
SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}}');
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}');
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}')
SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
```

View File

@@ -221,7 +221,7 @@ This function geocodes your data into point, or polygon, geometries for postal c
### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
Goecodes the postal code for a specified country into a **polygon** geometry.
Geocodes the postal code for a specified country into a **polygon** geometry.
#### Arguments
@@ -250,7 +250,7 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_postalcode_polygon('11211'
### cdb_geocode_postalcode_point(_code text, country_name text_)
Goecodes the postal code for a specified country into a **point** geometry.
Geocodes the postal code for a specified country into a **point** geometry.
#### Arguments

View File

@@ -8,7 +8,7 @@ ERB = erb
REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
@@ -16,8 +16,10 @@ REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
# These are the files to be installed in the server shared dir,
# for installation from scratch, upgrades and downgrades.
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(EXTENSION)--*.sql
OLD_VERSIONS = $(wildcard old_versions/*.sql)
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(EXTENSION)--*--*.sql \
$(OLD_VERSIONS)
SOURCES_DATA_DIR = sql/
SOURCES_DATA = $(wildcard sql/*.sql)

View File

@@ -0,0 +1,240 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -0,0 +1,223 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
-- isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.13.0'
default_version = '0.13.3'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

View File

@@ -0,0 +1,21 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,21 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;
$$ LANGUAGE plproxy;

View File

@@ -0,0 +1,37 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,48 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.1'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;

File diff suppressed because it is too large Load Diff

View File

@@ -4,12 +4,13 @@ CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], colt
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
)[0]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
TARGET cdb_observatory._OBS_ConnectUserTable;

View File

@@ -12,34 +12,6 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_internal_geocoder_config_{0}".format(username)

View File

@@ -7,12 +7,15 @@ RETURNS Geometry AS $$
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
plpy.debug('Requested geocoder is heremaps')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
plpy.debug('Requested geocoder is google')
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
plpy.debug('Requested geocoder is mapzen')
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
@@ -56,8 +59,8 @@ RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
@@ -137,13 +140,13 @@ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)

View File

@@ -70,15 +70,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:

View File

@@ -1,4 +1,27 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -6,18 +29,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
return result
$$ LANGUAGE plpythonu;
-- mapzen isodistance
@@ -25,11 +40,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(userna
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result

View File

@@ -1,4 +1,27 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -6,32 +29,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrones
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;

View File

@@ -77,8 +77,12 @@ class HereMapsRoutingIsoline:
isolines_response = parsed_response['response']['isoline']
isolines = []
for isoline in isolines_response:
if not isoline['component']:
geom_value = []
else:
geom_value = isoline['component'][0]['shape']
isolines.append({'range': isoline['range'],
'geom': isoline['component'][0]['shape']})
'geom': geom_value})
return isolines

View File

@@ -4,13 +4,18 @@ import plpy
def geo_polyline_to_multipolygon(polyline):
"""Convert a HERE polyline shape to a PostGIS multipolygon"""
coordinates = []
for point in polyline:
lat, lon = point.split(',')
coordinates.append("%s %s" % (lon, lat))
wkt_coordinates = ','.join(coordinates)
# In case we receive an empty polyline from here and we don't want to
# change this kind of thing in the extension sql
if not polyline:
sql = "SELECT ST_MPolyFromText(NULL, 4326) as geom"
else:
coordinates = []
for point in polyline:
lat, lon = point.split(',')
coordinates.append("%s %s" % (lon, lat))
wkt_coordinates = ','.join(coordinates)
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
geometry = plpy.execute(sql, 1)[0]['geom']
return geometry

View File

@@ -1,3 +1,3 @@
from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService

View File

@@ -104,17 +104,24 @@ class ObservatoryConfig(DataObservatoryConfig):
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY]
if not self._routing_provider:
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@property
def service_type(self):
return 'routing_mapzen'
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
@property
def mapzen_api_key(self):
@@ -128,93 +135,57 @@ class RoutingConfig(ServiceConfig):
def period_end_date(self):
return self._period_end_date
# Explicit class for the geocoder configuration for Mapzen
# which does not require users to be configured to use the service
class MapzenGeocoderConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._log_path = self._db_config.geocoder_log_path
try:
self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key
self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
self._cost_per_hit = 0
except Exception as e:
raise ConfigException("Malformed config for Mapzen geocoder: {0}".format(e))
@property
def service_type(self):
return 'geocoder_mapzen'
@property
def mapzen_api_key(self):
return self._mapzen_api_key
@property
def period_end_date(self):
return self._period_end_date
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def google_geocoder(self):
return None
@property
def geocoding_quota(self):
return self._monthly_quota
@property
def soft_geocoding_limit(self):
return False
@property
def is_high_resolution(self):
return True
@property
def log_path(self):
return self._log_path
class IsolinesRoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
'period_end_date', 'username', 'orgname',
'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
'geocoder_type']
'heremaps_isolines_app_id', 'geocoder_provider',
'heremaps_isolines_app_code', 'isolines_provider']
QUOTA_KEY = 'here_isolines_quota'
SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
PERIOD_END_DATE = 'period_end_date'
GEOCODER_TYPE_KEY = 'geocoder_type'
GOOGLE_GEOCODER = 'google'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'heremaps'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
filtered_config = {key: self._redis_config[key] for key in self.ROUTING_CONFIG_KEYS if key in self._redis_config.keys()}
filtered_config = {key: self._redis_config[key] for key in self.ISOLINES_CONFIG_KEYS if key in self._redis_config.keys()}
self.__parse_config(filtered_config, self._db_config)
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower()
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_isolines_limit = True
else:
self._isolines_provider = filtered_config[self.ISOLINES_PROVIDER_KEY].lower()
if not self._isolines_provider:
self._isolines_provider = self.DEFAULT_PROVIDER
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_isolines_limit = True
else:
self._soft_isolines_limit = False
elif self._isolines_provider == self.MAPZEN_PROVIDER:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._soft_isolines_limit = False
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
@property
def service_type(self):
return 'here_isolines'
if self._isolines_provider == self.HEREMAPS_PROVIDER:
return 'here_isolines'
elif self._isolines_provider == self.MAPZEN_PROVIDER:
return 'mapzen_isolines'
@property
def google_services_user(self):
return self._geocoder_provider == 'google'
@property
def isolines_quota(self):
@@ -236,45 +207,23 @@ class IsolinesRoutingConfig(ServiceConfig):
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def google_services_user(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
class MapzenIsolinesRoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
try:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
except Exception as e:
raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e))
@property
def service_type(self):
return 'mapzen_isolines'
@property
def isolines_quota(self):
return self._isolines_quota
@property
def soft_isolines_limit(self):
return False
@property
def period_end_date(self):
return self._period_end_date
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@property
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@property
def heremaps_provider(self):
return self._isolines_provider == self.HEREMAPS_PROVIDER
@property
def provider(self):
return self._isolines_provider
class InternalGeocoderConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -308,7 +257,7 @@ class GeocoderConfig(ServiceConfig):
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date',
'geocoder_provider', 'period_end_date',
'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
'mapzen_geocoder_api_key', 'username', 'orgname']
NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
@@ -320,12 +269,13 @@ class GeocoderConfig(ServiceConfig):
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
MAPZEN_GEOCODER = 'mapzen'
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
GEOCODER_TYPE = 'geocoder_type'
GEOCODER_PROVIDER = 'geocoder_provider'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(GeocoderConfig, self).__init__(redis_connection, db_conn,
@@ -335,21 +285,23 @@ class GeocoderConfig(ServiceConfig):
self.__check_config(filtered_config)
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if self._geocoder_provider == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
not self.heremaps_app_id or not self.heremaps_app_code:
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
if not self.mapzen_api_key:
raise ConfigException("""Mapzen config is not setted up""")
return True
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
if not self._geocoder_provider:
self._geocoder_provider = self.DEFAULT_PROVIDER
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._log_path = db_config.geocoder_log_path
@@ -357,39 +309,39 @@ class GeocoderConfig(ServiceConfig):
self._soft_geocoding_limit = True
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if self._geocoder_provider == self.NOKIA_GEOCODER:
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
self._cost_per_hit = 0
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota
self._cost_per_hit = 0
@property
def service_type(self):
if self._geocoder_type == self.GOOGLE_GEOCODER:
if self._geocoder_provider == self.GOOGLE_GEOCODER:
return 'geocoder_google'
elif self._geocoder_type == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
return 'geocoder_mapzen'
else:
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
@property
def heremaps_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
return self._geocoder_provider == self.NOKIA_GEOCODER
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
return self._geocoder_provider == self.GOOGLE_GEOCODER
@property
def mapzen_geocoder(self):
return self._geocoder_type == self.MAPZEN_GEOCODER
return self._geocoder_provider == self.MAPZEN_GEOCODER
@property
def google_client_id(self):
@@ -570,6 +522,9 @@ class ServicesRedisConfig:
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
ROUTING_PROVIDER_KEY = 'routing_provider'
def __init__(self, redis_conn):
self._redis_connection = redis_conn
@@ -582,7 +537,16 @@ class ServicesRedisConfig:
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
elif orgname:
# Not all the users have the provider key yet
if self.GEOCODER_PROVIDER_KEY not in user_config:
user_config[self.GEOCODER_PROVIDER_KEY] = ''
if self.ISOLINES_PROVIDER_KEY not in user_config:
user_config[self.ISOLINES_PROVIDER_KEY] = ''
if self.ROUTING_PROVIDER_KEY not in user_config:
user_config[self.ROUTING_PROVIDER_KEY] = ''
if orgname:
self.__get_organization_config(orgname, user_config)
return user_config
@@ -599,6 +563,15 @@ class ServicesRedisConfig:
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
if self.OBS_GENERAL_QUOTA_KEY in org_config:
user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
if self.PERIOD_END_DATE in org_config:
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
if self.GOOGLE_GEOCODER_CLIENT_ID in org_config:
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
if self.GOOGLE_GEOCODER_API_KEY in org_config:
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
if self.GEOCODER_PROVIDER_KEY in org_config:
user_config[self.GEOCODER_PROVIDER_KEY] = org_config[self.GEOCODER_PROVIDER_KEY]
if self.ISOLINES_PROVIDER_KEY in org_config:
user_config[self.ISOLINES_PROVIDER_KEY] = org_config[self.ISOLINES_PROVIDER_KEY]
if self.ROUTING_PROVIDER_KEY in org_config:
user_config[self.ROUTING_PROVIDER_KEY] = org_config[self.ROUTING_PROVIDER_KEY]

View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.7.0',
version='0.7.2.2',
description='CartoDB Services API Python Library',

View File

@@ -11,7 +11,9 @@ def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'geocoder_provider', service)
redis_conn.hset(user_redis_name, 'isolines_provider', service)
redis_conn.hset(user_redis_name, 'routing_provider', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
if do_quota:
redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota)

View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -109,6 +109,20 @@ class TestQuotaService(TestCase):
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_org_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user', orgname='testorg')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_obs_snapshot_quota_correctly(self):
qs = self.__build_obs_snapshot_quota_service('test_user')
qs.increment_success_service_use()
@@ -149,7 +163,7 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
def __build_routing_quota_service(self, username, service='routing_mapzen',
def __build_routing_quota_service(self, username, service='mapzen',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
@@ -158,6 +172,15 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
def __build_isolines_quota_service(self, username, service='mapzen',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock,
username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn)
def __build_obs_snapshot_quota_service(self, username, quota=100,
service='obs_snapshot',
orgname=None,