cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor python library to unify and rename as cdb_services

Browse Source
This commit is contained in:
Mario de Frutos
2016-01-29 11:54:50 +01:00
parent e7c58b9a51
commit 3960c13484
35 changed files with 397 additions and 872 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/extension/.gitignore vendored
View File

@@ -3,3 +3,4 @@ regression.diffs
regression.out
cdb_geocoder_server--0.0.1.sql
cdb_geocoder_server--0.1.0.sql
cdb_geocoder_server--0.2.0.sql

77
server/extension/cdb_geocoder_server--0.1.0--0.2.0.sql
View File

@@ -1,19 +1,76 @@
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_services.tools import RedisConnection
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@@ -35,16 +92,16 @@ $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from google_geocoder import googlemapsgeocoder
from cartodb_geocoder import quota_service
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])

57
server/extension/cdb_geocoder_server--0.2.0--0.1.0.sql
View File

@@ -1,3 +1,60 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder

638
server/extension/cdb_geocoder_server--0.2.0.sql
View File

@@ -1,638 +0,0 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from google_geocoder import googlemapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

6
server/extension/sql/0.2.0/10_redis_helper.sql
View File

@@ -32,19 +32,19 @@ RETURNS boolean AS $$
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
from cartodb_services.tools import RedisConnection
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],

4
server/extension/sql/0.2.0/15_config_helper.sql
View File

@@ -6,7 +6,7 @@ RETURNS boolean AS $$
return False
else:
import json
from cartodb_geocoder import config_helper
from cartodb_services.quota import GeocoderConfig
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
@@ -17,7 +17,7 @@ RETURNS boolean AS $$
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config

20
server/extension/sql/0.2.0/20_geocode_street.sql
View File

@@ -19,20 +19,20 @@ $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.quota import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@@ -54,16 +54,16 @@ $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from google_geocoder import googlemapsgeocoder
from cartodb_geocoder import quota_service
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.quota import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = googlemapsgeocoder.Geocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])

0
server/lib/python/cartodb_geocoder/cartodb_geocoder/__init__.py → server/lib/python/cartodb_services/cartodb_services/__init__.py
View File

1
server/lib/python/cartodb_services/cartodb_services/google/__init__.py Normal file
View File

@@ -0,0 +1 @@
from geocoder import GoogleMapsGeocoder

0
server/lib/python/heremaps/heremaps/exceptions.py → server/lib/python/cartodb_services/cartodb_services/google/exceptions.py
View File

31
server/lib/python/google_geocoder/google_geocoder/googlemapsgeocoder.py → server/lib/python/cartodb_services/cartodb_services/google/geocoder.py
View File

@@ -6,7 +6,7 @@ import googlemaps
from exceptions import MalformedResult
class Geocoder:
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
@@ -15,24 +15,25 @@ class Geocoder:
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
def geocode_address(self, searchtext, city=None, state=None,
country=None):
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext, components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
def _extract_lng_lat_from_result(self, result):
def geocode(self, searchtext, city=None, state=None,
country=None):
try:
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext,
components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
except KeyError:
raise MalformedResult()
def _extract_lng_lat_from_result(self, result):
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
def _build_optional_parameters(self, city=None, state=None,
country=None):
optional_params = {}

1
server/lib/python/cartodb_services/cartodb_services/here/__init__.py Normal file
View File

@@ -0,0 +1 @@
from geocoder import HereMapsGeocoder

21
server/lib/python/cartodb_services/cartodb_services/here/exceptions.py Normal file
View File

@@ -0,0 +1,21 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class BadGeocodingParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Bad geocoding params: ' + json.dumps(self.value))
class NoGeocodingParams(Exception):
def __str__(self):
return repr('No params for geocoding specified')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

19
server/lib/python/heremaps/heremaps/heremapsgeocoder.py → server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@@ -4,13 +4,10 @@
import json
import requests
from heremaps.exception import BadGeocodingParams
from heremaps.exception import EmptyGeocoderResponse
from heremaps.exception import NoGeocodingParams
from heremaps.exception import MalformedResult
from exceptions import *
class Geocoder:
class HereMapsGeocoder:
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@@ -65,17 +62,19 @@ class Geocoder:
params[key] = value
if not params:
raise NoGeocodingParams()
return self.geocode(params)
return self._execute_geocode(params)
def _execute_geocode(self, params):
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
raise BadGeocodingParams(params)
try:
response = self._perform_request(params)
results = response['Response']['View'][0]['Result']
results = response['Response']['View'][0]['Result'][0]
return self._extract_lng_lat_from_result(results)
except IndexError:
return []
except KeyError:
raise MalformedResult()
def _perform_request(self, params):
request_params = {
@@ -92,11 +91,7 @@ class Geocoder:
response.raise_for_status()
def _extract_lng_lat_from_result(self, result):
try:
location = result['Location']
except KeyError:
raise MalformedResult()
location = result['Location']
longitude = location['DisplayPosition']['Longitude']
latitude = location['DisplayPosition']['Latitude']

3
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from config import GeocoderConfig, ConfigException
from quota import QuotaService
from user import UserMetricsService

0
server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py → server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

4
server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py → server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -1,4 +1,4 @@
import user_service
from user import UserMetricsService
from datetime import date
@@ -8,7 +8,7 @@ class QuotaService:
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._user_service = user_service.UserService(
self._user_service = UserMetricsService(
self._user_geocoder_config, redis_connection)
def check_user_quota(self):

2
server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py → server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@@ -2,7 +2,7 @@ from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
class UserService:
class UserMetricsService:
""" Class to manage all the user info """
SERVICE_GEOCODER_NOKIA = 'geocoder_here'

1
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py Normal file
View File

@@ -0,0 +1 @@
from redis_tools import RedisConnection

2
server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py → server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py
View File

@@ -1,7 +1,7 @@
from redis.sentinel import Sentinel
class RedisHelper:
class RedisConnection:
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds

8
server/lib/python/cartodb_geocoder/requirements.txt → server/lib/python/cartodb_services/requirements.txt
View File

@@ -1,11 +1,13 @@
redis==2.10.5
hiredis==0.1.5
# Dependency with incsv in the import
python-dateutil==2.2
heremaps==0.0.1
googlegeocoder==0.0.1
googlemaps
googlemaps==2.4.2
# Dependency for googlemaps package
requests<=2.9.1
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7
requests-mock==0.7.0

10
server/lib/python/cartodb_geocoder/setup.py → server/lib/python/cartodb_services/setup.py
View File

@@ -1,5 +1,5 @@
"""
CartoDB Geocoder Python Library
CartoDB Services Python Library
See:
https://github.com/CartoDB/geocoder-api
@@ -8,11 +8,11 @@ https://github.com/CartoDB/geocoder-api
from setuptools import setup, find_packages
setup(
name='cartodb_geocoder',
name='cartodb_services',
version='0.0.1',
version='0.1.0',
description='CartoDB Geocoder Python Library',
description='CartoDB Services API Python Library',
url='https://github.com/CartoDB/geocoder-api',
@@ -29,7 +29,7 @@ setup(
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools geocoder',
keywords='maps api mapping tools geocoder routing',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),

20
server/lib/python/cartodb_geocoder/test/test_config_helper.py → server/lib/python/cartodb_services/test/test_config.py
View File

@@ -1,41 +1,41 @@
import test_helper
from cartodb_geocoder import config_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfigHelper(TestCase):
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(config_helper.ConfigException,
config_helper.GeocoderConfig,
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

119
server/lib/python/cartodb_services/test/test_googlegeocoder.py Normal file
View File

@@ -0,0 +1,119 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
from cartodb_services.google.exceptions import NoGeocodingParams
from cartodb_services.google.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json'
EMPTY_RESPONSE = """{
"results" : [],
"status" : "ZERO_RESULTS"
}"""
GOOD_RESPONSE = """{
"results" : [
{
"address_components" : [
{
"long_name" : "1600",
"short_name" : "1600",
"types" : [ "street_number" ]
},
{
"long_name" : "Amphitheatre Pkwy",
"short_name" : "Amphitheatre Pkwy",
"types" : [ "route" ]
},
{
"long_name" : "Mountain View",
"short_name" : "Mountain View",
"types" : [ "locality", "political" ]
},
{
"long_name" : "Santa Clara County",
"short_name" : "Santa Clara County",
"types" : [ "administrative_area_level_2", "political" ]
},
{
"long_name" : "California",
"short_name" : "CA",
"types" : [ "administrative_area_level_1", "political" ]
},
{
"long_name" : "United States",
"short_name" : "US",
"types" : [ "country", "political" ]
},
{
"long_name" : "94043",
"short_name" : "94043",
"types" : [ "postal_code" ]
}
],
"formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
},
"location_type" : "ROOFTOP",
"viewport" : {
"northeast" : {
"lat" : 37.4238253802915,
"lng" : -122.0829009197085
},
"southwest" : {
"lat" : 37.4211274197085,
"lng" : -122.0855988802915
}
}
},
"place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types" : [ "street_address" ]
}
],
"status" : "OK"
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -122.0842499)
self.assertEqual(response[1], 37.4224764)
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

0
server/lib/python/cartodb_geocoder/test/test_helper.py → server/lib/python/cartodb_services/test/test_helper.py
View File

84
server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py → server/lib/python/cartodb_services/test/test_heremapsgeocoder.py
View File

@@ -2,25 +2,28 @@
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from heremaps import heremapsgeocoder
from heremaps.heremapsexceptions import BadGeocodingParams
from heremaps.heremapsexceptions import EmptyGeocoderResponse
from heremaps.heremapsexceptions import NoGeocodingParams
from heremaps.heremapsexceptions import MalformedResult
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
from cartodb_services.here.exceptions import NoGeocodingParams
from cartodb_services.here.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
class GeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = {
@requests_mock.Mocker()
class HereMapsGeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:31:57.273+0000"
},
"View": []
}
}
}"""
GOOD_RESPONSE = {
GOOD_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:30:32.187+0000"
@@ -28,7 +31,7 @@ class GeocoderTestCase(unittest.TestCase):
"View": [{
"_type": "SearchResultsViewType",
"ViewId": 0,
"Result": [{
"Result": {
"Relevance": 0.89,
"MatchLevel": "street",
"MatchQuality": {
@@ -57,7 +60,7 @@ class GeocoderTestCase(unittest.TestCase):
}
},
"Address": {
"Label": "Calle de Eloy Gonzalo, Madrid, España",
"Label": "Calle de Eloy Gonzalo, Madrid, Espana",
"Country": "ESP",
"State": "Comunidad de Madrid",
"County": "Madrid",
@@ -65,7 +68,7 @@ class GeocoderTestCase(unittest.TestCase):
"District": "Trafalgar",
"Street": "Calle de Eloy Gonzalo",
"AdditionalData": [{
"value": "España",
"value": "Espana",
"key": "CountryName"
},
{
@@ -78,45 +81,52 @@ class GeocoderTestCase(unittest.TestCase):
}]
}
}
}]
}
}]
}
}
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = heremapsgeocoder.Geocoder(None, None)
self.geocoder = HereMapsGeocoder(None, None)
def test_geocode_address_with_valid_params(self):
self.geocoder.perform_request = lambda x: self.GOOD_RESPONSE
response = self.geocoder.geocode_address(
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
def test_geocode_address_with_invalid_params(self):
self.assertEqual(response[0], -3.70126)
self.assertEqual(response[1], 40.43433)
def test_geocode_address_with_invalid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(BadGeocodingParams):
self.geocoder.geocode_address(
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
manolo='escobar')
def test_geocode_address_with_no_params(self):
def test_geocode_address_with_no_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(NoGeocodingParams):
self.geocoder.geocode_address()
self.geocoder.geocode()
def test_geocode_address_empty_response(self):
self.geocoder.perform_request = lambda x: self.EMPTY_RESPONSE
with self.assertRaises(EmptyGeocoderResponse):
self.geocoder.geocode_address(searchtext='lkajfñlasjfñ')
def test_extract_lng_lat_from_result(self):
result = self.GOOD_RESPONSE['Response']['View'][0]['Result'][0]
coordinates = self.geocoder.extract_lng_lat_from_result(result)
self.assertEqual(coordinates[0], -3.70126)
self.assertEqual(coordinates[1], 40.43433)
def test_extract_lng_lat_from_result_with_malformed_result(self):
result = {'manolo': 'escobar'}
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.extract_lng_lat_from_result(result)
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

35
server/lib/python/cartodb_geocoder/test/test_quota_service.py → server/lib/python/cartodb_services/test/test_quota_service.py
View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import quota_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -14,10 +14,6 @@ class TestQuotaService(TestCase):
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
# def increment_geocoder_uses(self, username, orgname=None,
# date=date.today(), service='geocoder_here',
# metric='success_responses', amount=20):
def setUp(self):
self.redis_conn = MockRedis()
@@ -68,28 +64,19 @@ class TestQuotaService(TestCase):
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
total_requests = self.redis_conn.zscore(name, date.today().day)
assert total_requests == 3
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month)
org_total_requests = self.redis_conn.zscore(org_name, date.today().day)
assert org_total_requests == 3
user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
user_total_requests = self.redis_conn.zscore(user_name, date.today().day)
assert user_total_requests == 3
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
@@ -101,9 +88,9 @@ class TestQuotaService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return quota_service.QuotaService(geocoder_config,
redis_connection = self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

12
server/lib/python/cartodb_geocoder/test/test_user_service.py → server/lib/python/cartodb_services/test/test_user_service.py
View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import user_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
@@ -84,7 +84,7 @@ class TestUserService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return user_service.UserService(geocoder_config, self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return UserMetricsService(geocoder_config, self.redis_conn)

0
server/lib/python/google_geocoder/google_geocoder/__init__.py
View File

7
server/lib/python/google_geocoder/google_geocoder/exceptions.py
View File

@@ -1,7 +0,0 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
class MalformedResult(Exception):
def __str__(self):
return repr('Malformed result. The API might have changed.')

41
server/lib/python/google_geocoder/setup.py
View File

@@ -1,41 +0,0 @@
"""
A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension
See:
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='google-geocoder',
version='0.0.1',
description='A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 2 - Beta',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest'],
}
)

0
server/lib/python/heremaps/heremaps/__init__.py
View File

0
server/lib/python/heremaps/heremaps/tests/__init__.py
View File

4
server/lib/python/heremaps/requirements.txt
View File

@@ -1,4 +0,0 @@
requests==2.9.1
# Test
nose==1.3.7

41
server/lib/python/heremaps/setup.py
View File

@@ -1,41 +0,0 @@
"""
A Here Maps API Python wrapper
See:
https://developer.here.com
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='heremaps',
version='0.0.1',
description='A Here Maps API Python wrapper',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 5 - Production',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest'],
}
)