cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #236 from CartoDB/development

Browse Source 
Release 0.13.3
This commit is contained in:
Mario de Frutos
2016-07-25 13:26:27 +02:00
committed by GitHub
parent 15eb8633d7 d0f8797ccb
commit 5c5bed5557
18 changed files with 2942 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/extension/Makefile
+1 -1
View File
@@ -8,7 +8,7 @@ ERB = erb
REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
server/extension/cdb_dataservices_server--0.13.2--0.13.3.sql
+240
View File
@@ -0,0 +1,240 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.notice('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.notice('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.notice('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.notice('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
server/extension/cdb_dataservices_server--0.13.3--0.13.2.sql
+223
View File
@@ -0,0 +1,223 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
-- isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = isoline
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
server/extension/cdb_dataservices_server--0.13.3.sql
+2297
View File
File diff suppressed because it is too large Load Diff
server/extension/cdb_dataservices_server.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.13.2'
default_version = '0.13.3'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server
server/extension/cdb_dataservices_server--0.13.1--0.13.2.sql → server/extension/old_versions/cdb_dataservices_server--0.13.1--0.13.2.sql
View File
server/extension/cdb_dataservices_server--0.13.2--0.13.1.sql → server/extension/old_versions/cdb_dataservices_server--0.13.2--0.13.1.sql
View File
server/extension/cdb_dataservices_server--0.13.2.sql → server/extension/old_versions/cdb_dataservices_server--0.13.2.sql
View File
server/extension/sql/15_config_helper.sql
-28
View File
@@ -12,34 +12,6 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_internal_geocoder_config_{0}".format(username)
server/extension/sql/20_geocode_street.sql
+8 -5
View File
@@ -7,12 +7,15 @@ RETURNS Geometry AS $$
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
plpy.notice('Requested geocoder is heremaps')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
plpy.notice('Requested geocoder is google')
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
plpy.notice('Requested geocoder is mapzen')
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
@@ -56,8 +59,8 @@ RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
@@ -137,13 +140,13 @@ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
server/extension/sql/80_isolines_helper.sql
+3 -3
View File
@@ -70,15 +70,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
server/extension/sql/85_isodistance.sql
+24 -4
View File
@@ -4,11 +4,31 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.notice('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.notice('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
@@ -20,11 +40,11 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(userna
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
server/extension/sql/90_isochrone.sql
+28 -9
View File
@@ -1,4 +1,27 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.notice('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options], 1)
elif user_isolines_config.mapzen_provider:
plpy.notice('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options], 1)
else:
plpy.error('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -6,26 +29,22 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
-- mapzen isochrones
-- mapzen isochrone
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
return result
$$ LANGUAGE plpythonu;
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
+87 -123
View File
@@ -104,17 +104,24 @@ class ObservatoryConfig(DataObservatoryConfig):
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY]
if not self._routing_provider:
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@property
def service_type(self):
return 'routing_mapzen'
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
@property
def mapzen_api_key(self):
@@ -128,93 +135,57 @@ class RoutingConfig(ServiceConfig):
def period_end_date(self):
return self._period_end_date
# Explicit class for the geocoder configuration for Mapzen
# which does not require users to be configured to use the service
class MapzenGeocoderConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._log_path = self._db_config.geocoder_log_path
try:
self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key
self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
self._cost_per_hit = 0
except Exception as e:
raise ConfigException("Malformed config for Mapzen geocoder: {0}".format(e))
@property
def service_type(self):
return 'geocoder_mapzen'
@property
def mapzen_api_key(self):
return self._mapzen_api_key
@property
def period_end_date(self):
return self._period_end_date
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def google_geocoder(self):
return None
@property
def geocoding_quota(self):
return self._monthly_quota
@property
def soft_geocoding_limit(self):
return False
@property
def is_high_resolution(self):
return True
@property
def log_path(self):
return self._log_path
class IsolinesRoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
'period_end_date', 'username', 'orgname',
'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
'geocoder_type']
'heremaps_isolines_app_id', 'geocoder_provider',
'heremaps_isolines_app_code', 'isolines_provider']
QUOTA_KEY = 'here_isolines_quota'
SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
PERIOD_END_DATE = 'period_end_date'
GEOCODER_TYPE_KEY = 'geocoder_type'
GOOGLE_GEOCODER = 'google'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'heremaps'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
filtered_config = {key: self._redis_config[key] for key in self.ROUTING_CONFIG_KEYS if key in self._redis_config.keys()}
filtered_config = {key: self._redis_config[key] for key in self.ISOLINES_CONFIG_KEYS if key in self._redis_config.keys()}
self.__parse_config(filtered_config, self._db_config)
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower()
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_isolines_limit = True
else:
self._isolines_provider = filtered_config[self.ISOLINES_PROVIDER_KEY].lower()
if not self._isolines_provider:
self._isolines_provider = self.DEFAULT_PROVIDER
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_isolines_limit = True
else:
self._soft_isolines_limit = False
elif self._isolines_provider == self.MAPZEN_PROVIDER:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._soft_isolines_limit = False
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
@property
def service_type(self):
return 'here_isolines'
if self._isolines_provider == self.HEREMAPS_PROVIDER:
return 'here_isolines'
elif self._isolines_provider == self.MAPZEN_PROVIDER:
return 'mapzen_isolines'
@property
def google_services_user(self):
return self._geocoder_provider == 'google'
@property
def isolines_quota(self):
@@ -236,45 +207,23 @@ class IsolinesRoutingConfig(ServiceConfig):
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def google_services_user(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
class MapzenIsolinesRoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
try:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
except Exception as e:
raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e))
@property
def service_type(self):
return 'mapzen_isolines'
@property
def isolines_quota(self):
return self._isolines_quota
@property
def soft_isolines_limit(self):
return False
@property
def period_end_date(self):
return self._period_end_date
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@property
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@property
def heremaps_provider(self):
return self._isolines_provider == self.HEREMAPS_PROVIDER
@property
def provider(self):
return self._isolines_provider
class InternalGeocoderConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -308,7 +257,7 @@ class GeocoderConfig(ServiceConfig):
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date',
'geocoder_provider', 'period_end_date',
'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
'mapzen_geocoder_api_key', 'username', 'orgname']
NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
@@ -320,12 +269,13 @@ class GeocoderConfig(ServiceConfig):
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
MAPZEN_GEOCODER = 'mapzen'
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
GEOCODER_TYPE = 'geocoder_type'
GEOCODER_PROVIDER = 'geocoder_provider'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(GeocoderConfig, self).__init__(redis_connection, db_conn,
@@ -335,21 +285,23 @@ class GeocoderConfig(ServiceConfig):
self.__check_config(filtered_config)
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if self._geocoder_provider == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
not self.heremaps_app_id or not self.heremaps_app_code:
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
if not self.mapzen_api_key:
raise ConfigException("""Mapzen config is not setted up""")
return True
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
if not self._geocoder_provider:
self._geocoder_provider = self.DEFAULT_PROVIDER
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._log_path = db_config.geocoder_log_path
@@ -357,39 +309,39 @@ class GeocoderConfig(ServiceConfig):
self._soft_geocoding_limit = True
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if self._geocoder_provider == self.NOKIA_GEOCODER:
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
self._cost_per_hit = 0
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota
self._cost_per_hit = 0
@property
def service_type(self):
if self._geocoder_type == self.GOOGLE_GEOCODER:
if self._geocoder_provider == self.GOOGLE_GEOCODER:
return 'geocoder_google'
elif self._geocoder_type == self.MAPZEN_GEOCODER:
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
return 'geocoder_mapzen'
else:
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
@property
def heremaps_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
return self._geocoder_provider == self.NOKIA_GEOCODER
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
return self._geocoder_provider == self.GOOGLE_GEOCODER
@property
def mapzen_geocoder(self):
return self._geocoder_type == self.MAPZEN_GEOCODER
return self._geocoder_provider == self.MAPZEN_GEOCODER
@property
def google_client_id(self):
@@ -570,6 +522,9 @@ class ServicesRedisConfig:
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
PERIOD_END_DATE = 'period_end_date'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
ISOLINES_PROVIDER_KEY = 'isolines_provider'
ROUTING_PROVIDER_KEY = 'routing_provider'
def __init__(self, redis_conn):
self._redis_connection = redis_conn
@@ -599,6 +554,15 @@ class ServicesRedisConfig:
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
if self.OBS_GENERAL_QUOTA_KEY in org_config:
user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
if self.PERIOD_END_DATE in org_config:
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
if self.GOOGLE_GEOCODER_CLIENT_ID in org_config:
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
if self.GOOGLE_GEOCODER_API_KEY in org_config:
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
if self.GEOCODER_PROVIDER_KEY in org_config:
user_config[self.GEOCODER_PROVIDER_KEY] = org_config[self.GEOCODER_PROVIDER_KEY]
if self.ISOLINES_PROVIDER_KEY in org_config:
user_config[self.ISOLINES_PROVIDER_KEY] = org_config[self.ISOLINES_PROVIDER_KEY]
if self.ROUTING_PROVIDER_KEY in org_config:
user_config[self.ROUTING_PROVIDER_KEY] = org_config[self.ROUTING_PROVIDER_KEY]
server/lib/python/cartodb_services/setup.py
+1 -1
View File
@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.7.1',
version='0.7.2',
description='CartoDB Services API Python Library',
server/lib/python/cartodb_services/test/test_helper.py
+3 -1
View File
@@ -11,7 +11,9 @@ def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'geocoder_provider', service)
redis_conn.hset(user_redis_name, 'isolines_provider', service)
redis_conn.hset(user_redis_name, 'routing_provider', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
if do_quota:
redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota)
server/lib/python/cartodb_services/test/test_quota_service.py
+25 -2
View File
@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -109,6 +109,20 @@ class TestQuotaService(TestCase):
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_org_isolines_quota_correctly(self):
qs = self.__build_isolines_quota_service('test_user', orgname='testorg')
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_service_use(amount=1500000)
assert qs.check_user_quota() is False
def test_should_check_user_obs_snapshot_quota_correctly(self):
qs = self.__build_obs_snapshot_quota_service('test_user')
qs.increment_success_service_use()
@@ -149,7 +163,7 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
def __build_routing_quota_service(self, username, service='routing_mapzen',
def __build_routing_quota_service(self, username, service='mapzen',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
@@ -158,6 +172,15 @@ class TestQuotaService(TestCase):
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
def __build_isolines_quota_service(self, username, service='mapzen',
orgname=None, soft_limit=False,
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock,
username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn)
def __build_obs_snapshot_quota_service(self, username, quota=100,
service='obs_snapshot',
orgname=None,