cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #248 from CartoDB/rollbar_integration

Browse Source 
Improve logger capability
This commit is contained in:
Mario de Frutos
2016-08-03 12:17:26 +02:00
committed by GitHub
parent 4d47b905e8 a33999ed04
commit fcee6f440c
42 changed files with 563 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -77,8 +77,11 @@ Steps to deploy a new Data Services API version :
SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}');
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}')
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}')
SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
# Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior)
SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}')
```
- configure the user DB:

2
server/extension/Makefile
View File

@@ -10,7 +10,7 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
# DATA is a special variable used by postgres build infrastructure
# These are the files to be installed in the server shared dir,

0
server/extension/cdb_dataservices_server--0.13.3.1--0.14.sql Normal file
View File

0
server/extension/cdb_dataservices_server--0.14--0.13.3.1.sql Normal file
View File

2
server/extension/cdb_dataservices_server.control
View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.13.3.1'
default_version = '0.14'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.13.3--0.13.3.1.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3--0.13.3.1.sql
View File

0
server/extension/cdb_dataservices_server--0.13.3.1--0.13.3.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3.1--0.13.3.sql
View File

0
server/extension/cdb_dataservices_server--0.13.3.1.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3.1.sql
View File

18
server/extension/sql/100_routing_helper.sql
View File

@@ -17,19 +17,23 @@ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MapzenRouting(user_routing_config.mapzen_api_key)
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
if not waypoints or len(waypoints) < 2:
plpy.notice("Empty origin or destination")
logger.notice("Empty origin or destination")
quota_service.increment_empty_service_use()
return [None, None, None]
@@ -52,12 +56,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$
quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to calculate mapzen routing')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

130
server/extension/sql/110_data_observatory_augmentation.sql
View File

@@ -35,6 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -42,9 +43,11 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
@@ -56,12 +59,10 @@ RETURNS json AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -85,15 +86,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
@@ -109,12 +113,10 @@ RETURNS SETOF JSON AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -136,6 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@@ -143,9 +146,11 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@@ -157,12 +162,10 @@ RETURNS json AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -184,15 +187,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@@ -208,12 +214,10 @@ RETURNS SETOF JSON AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -241,15 +245,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
@@ -261,12 +268,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -292,15 +297,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@@ -312,12 +320,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -345,15 +351,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
@@ -365,12 +374,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -396,15 +403,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@@ -416,12 +426,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -447,15 +455,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@@ -467,12 +478,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -498,15 +507,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
@@ -518,12 +530,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

26
server/extension/sql/115_data_observatory_exploration.sql
View File

@@ -15,15 +15,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
@@ -43,12 +46,10 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -70,15 +71,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@@ -97,12 +101,10 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

91
server/extension/sql/120_data_observatory_geometries.sql
View File

@@ -17,15 +17,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
@@ -37,12 +40,10 @@ RETURNS geometry(Geometry, 4326) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -66,15 +67,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
@@ -86,12 +90,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use obs_search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -115,15 +117,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
@@ -135,12 +140,10 @@ RETURNS geometry(Geometry, 4326) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -166,15 +169,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
@@ -191,12 +197,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -224,15 +228,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
@@ -249,12 +256,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -280,15 +285,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
@@ -305,12 +313,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -338,15 +344,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
@@ -363,12 +372,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

54
server/extension/sql/20_geocode_street.sql
View File

@@ -7,19 +7,16 @@ RETURNS Geometry AS $$
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
plpy.debug('Requested geocoder is heremaps')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
plpy.debug('Requested geocoder is google')
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
plpy.debug('Requested geocoder is mapzen')
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
@@ -35,7 +32,7 @@ RETURNS Geometry AS $$
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Here geocoder is not available for your account.')
raise Exception('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@@ -50,7 +47,7 @@ RETURNS Geometry AS $$
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Google geocoder is not available for your account.')
raise Exception('Google geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@@ -71,17 +68,20 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@@ -92,12 +92,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using here maps')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -106,13 +104,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@@ -123,12 +125,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using google maps')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -138,15 +138,19 @@ RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
@@ -162,12 +166,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

11
server/extension/sql/30_admin0.sql
View File

@@ -2,12 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
@@ -20,12 +23,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

22
server/extension/sql/40_admin1.sql
View File

@@ -3,12 +3,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
@@ -21,12 +24,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -36,12 +37,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
@@ -54,12 +58,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

34
server/extension/sql/50_namedplaces.sql
View File

@@ -3,12 +3,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
@@ -21,12 +25,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -36,12 +38,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
@@ -54,12 +59,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -69,12 +72,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
@@ -87,12 +93,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

44
server/extension/sql/60_postalcodes.sql
View File

@@ -2,12 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
@@ -20,12 +23,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -34,12 +35,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
@@ -52,12 +56,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -66,12 +68,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
@@ -84,12 +89,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@@ -98,12 +101,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
@@ -116,12 +122,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

11
server/extension/sql/70_ips.sql
View File

@@ -2,12 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(u
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
@@ -20,12 +23,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

37
server/extension/sql/80_isolines_helper.sql
View File

@@ -6,17 +6,21 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL)
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
user_isolines_routing_config.heremaps_app_code, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -44,12 +48,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
@@ -65,21 +67,23 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.mapzen import MatrixClient, MapzenIsolines
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
mapzen_isolines = MapzenIsolines(client, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@@ -117,13 +121,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get mapzen isolines')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

6
server/extension/sql/85_isodistance.sql
View File

@@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance

6
server/extension/sql/90_isochrone.sql
View File

@@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone

3
server/lib/python/cartodb_services/cartodb_services/google/geocoder.py
View File

@@ -9,11 +9,12 @@ from exceptions import MalformedResult
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
def __init__(self, client_id, client_secret, logger):
self.client_id = self._clean_client_id(client_id)
self.client_secret = client_secret
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
self._logger = logger
def geocode(self, searchtext, city=None, state=None,
country=None):

11
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@@ -47,10 +47,11 @@ class HereMapsGeocoder:
'strictlanguagemode'
] + ADDRESS_PARAMS
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
self.app_id = app_id
self.app_code = app_code
self._logger = logger
self.maxresults = maxresults
self.gen = gen
self.host = host
@@ -88,9 +89,15 @@ class HereMapsGeocoder:
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:
self._logger.warning('Error 4xx trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
return []
else:
response.raise_for_status()
self._logger.error('Error trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
raise Exception('Error trying to geocode street using Here')
def _extract_lng_lat_from_result(self, result):
location = result['Location']

11
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@@ -25,9 +25,11 @@ class HereMapsRoutingIsoline:
'quality'
]
def __init__(self, app_id, app_code, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_id, app_code, logger,
base_url=PRODUCTION_ROUTING_BASE_URL):
self._app_id = app_id
self._app_code = app_code
self._logger = logger
self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH)
def calculate_isodistance(self, source, mode, data_range, options=[]):
@@ -54,7 +56,12 @@ class HereMapsRoutingIsoline:
elif response.status_code == requests.codes.bad_request:
return []
else:
response.raise_for_status()
self._logger.error('Error trying to calculate HERE isolines',
data={"response": response.json(), "source":
source, "mode": mode, "data_range":
data_range, "range_type": range_type,
"options": options})
raise Exception('Error trying to calculate HERE isolines')
def __parse_options(self, options):
return dict(option.split('=') for option in options)

5
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@@ -12,9 +12,10 @@ class MapzenGeocoder:
BASE_URL = 'https://search.mapzen.com/v1/search'
def __init__(self, app_key, base_url=BASE_URL):
def __init__(self, app_key, logger, base_url=BASE_URL):
self._app_key = app_key
self._url = base_url
self._logger = logger
@qps_retry
def geocode(self, searchtext, city=None, state_province=None, country=None):
@@ -25,7 +26,7 @@ class MapzenGeocoder:
elif response.status_code == requests.codes.bad_request:
return []
else:
response.raise_for_status()
raise Exception('Error trying to geocode {0} using mapzen'.format(searchtext))
def _build_requests_parameters(self, searchtext, city=None,
state_province=None, country=None):

13
server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py
View File

@@ -1,5 +1,5 @@
from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
import logging
class MapzenIsolines:
@@ -9,8 +9,9 @@ class MapzenIsolines:
EARTH_RADIUS_METERS = 6367444
def __init__(self, matrix_client):
def __init__(self, matrix_client, logger):
self._matrix_client = matrix_client
self._logger = logger
"""Get an isochrone using mapzen API.
@@ -85,11 +86,7 @@ class MapzenIsolines:
def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
# NOTE: not for production
#logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
#logging.basicConfig(level=logging.DEBUG)
logging.debug('origin = %s' % origin)
logging.debug('costing_model = %s' % costing_model)
logging.debug('isorange = %d' % isorange)
self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange})
# Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
# we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
@@ -114,7 +111,7 @@ class MapzenIsolines:
else:
costs[idx] = isorange
logging.debug('i = %d, costs = %s' % (i, costs))
# self._logger.debug('i = %d, costs = %s' % (i, costs))
errors = [(cost - isorange) / float(isorange) for cost in costs]
max_abs_error = max([abs(e) for e in errors])

16
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
View File

@@ -2,6 +2,7 @@ import requests
import json
from qps import qps_retry
class MatrixClient:
"""
@@ -18,8 +19,9 @@ class MatrixClient:
ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
def __init__(self, matrix_key):
def __init__(self, matrix_key, logger):
self._matrix_key = matrix_key
self._logger = logger
"""Get distances and times to a set of locations.
See https://mapzen.com/documentation/matrix/api-reference/
@@ -40,6 +42,16 @@ class MatrixClient:
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
response.raise_for_status() # raise exception if not 200 OK
raise Exception('testing exception handling')
if not requests.codes.ok:
self._logger.error('Error trying to get matrix distance from mapzen',
data={"response": response.json(), "locations":
locations, "costing": costing})
raise Exception('Error trying to get matrix distance from mapzen')
else:
self._logger.debug('Done get matrix distance from mapzen',
data={"response": response.json(),
"locations": locations,
"costing": costing})
return response.json()

9
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@@ -28,9 +28,10 @@ class MapzenRouting:
METRICS_UNITS = 'kilometers'
IMPERIAL_UNITS = 'miles'
def __init__(self, app_key, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL):
self._app_key = app_key
self._url = base_url
self._logger = logger
@qps_retry
def calculate_route_point_to_point(self, waypoints, mode,
@@ -48,7 +49,11 @@ class MapzenRouting:
elif response.status_code == requests.codes.bad_request:
return MapzenRoutingResponse(None, None, None)
else:
response.raise_for_status()
self._logger.error('Error trying to calculate route using HERE',
data={"response": response.json(), "waypoints":
waypoints, "mode": mode, "options":
options})
raise Exception('Error trying to calculate route using HERE')
def __parse_options(self, options):
return dict(option.split('=') for option in options)

27
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -15,6 +15,7 @@ class ServiceConfig(object):
self._username = username
self._orgname = orgname
self._db_config = ServicesDBConfig(db_conn, username, orgname)
self._environment = self._db_config._server_environment
if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build(
username, orgname)
@@ -33,6 +34,11 @@ class ServiceConfig(object):
def organization(self):
return self._orgname
@property
def environment(self):
return self._environment
class DataObservatoryConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -55,6 +61,7 @@ class DataObservatoryConfig(ServiceConfig):
def connection_str(self):
return self._connection_str
class ObservatorySnapshotConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
@@ -78,6 +85,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig):
def service_type(self):
return 'obs_snapshot'
class ObservatoryConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_general_limit'
@@ -101,6 +109,7 @@ class ObservatoryConfig(DataObservatoryConfig):
def service_type(self):
return 'obs_general'
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
@@ -162,6 +171,7 @@ class IsolinesRoutingConfig(ServiceConfig):
if not self._isolines_provider:
self._isolines_provider = self.DEFAULT_PROVIDER
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._heremaps_app_id = db_config.heremaps_isolines_app_id
@@ -174,7 +184,6 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._soft_isolines_limit = False
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
@property
def service_type(self):
@@ -400,11 +409,23 @@ class ServicesDBConfig:
return self._build()
def _build(self):
self._get_server_config()
self._get_here_config()
self._get_mapzen_config()
self._get_logger_config()
self._get_data_observatory_config()
def _get_server_config(self):
server_config_json = self._get_conf('server_conf')
if not server_config_json:
self._server_environment = 'development'
else:
server_config_json = json.loads(server_config_json)
if 'environment' in server_config_json:
self._server_environment = server_config_json['environment']
else:
self._server_environment = 'development'
def _get_here_config(self):
heremaps_conf_json = self._get_conf('heremaps_conf')
if not heremaps_conf_json:
@@ -460,6 +481,10 @@ class ServicesDBConfig:
except Exception as e:
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
@property
def server_environment(self):
return self._server_environment
@property
def heremaps_isolines_app_id(self):
return self._heremaps_isolines_app_id

10
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@@ -4,17 +4,17 @@ import json
import re
class LoggerFactory:
class MetricsLoggerFactory:
@classmethod
def build(self, service_config):
if re.match('geocoder_*', service_config.service_type):
return GeocoderLogger(service_config)
return MetricsGeocoderLogger(service_config)
else:
return None
class Logger(object):
class MetricsLogger(object):
__metaclass__ = abc.ABCMeta
def __init__(self, file_path):
@@ -30,10 +30,10 @@ class Logger(object):
raise NotImplementedError('log method must be defined')
class GeocoderLogger(Logger):
class MetricsGeocoderLogger(MetricsLogger):
def __init__(self, service_config):
super(GeocoderLogger, self).__init__(service_config.log_path)
super(MetricsGeocoderLogger, self).__init__(service_config.log_path)
self._service_config = service_config
def log(self, **data):

10
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -1,5 +1,5 @@
from user import UserMetricsService
from log import LoggerFactory
from log import MetricsLoggerFactory
from datetime import date
import re
@@ -14,7 +14,7 @@ class QuotaService:
redis_connection)
self._user_service = UserMetricsService(self._user_service_config,
redis_connection)
self._logger = LoggerFactory.build(user_service_config)
self._metrics_logger = MetricsLoggerFactory.build(user_service_config)
def check_user_quota(self):
return self._quota_checker.check()
@@ -48,11 +48,11 @@ class QuotaService:
amount=amount)
def _log_service_process(self, event):
if self._logger:
if self._metrics_logger:
if event is 'success' or event is 'empty':
self._logger.log(success=True)
self._metrics_logger.log(success=True)
elif event is 'empty':
self._logger.log(success=False)
self._metrics_logger.log(success=False)
class QuotaChecker:

1
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@@ -1,3 +1,4 @@
from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate
from polyline import PolyLine
from log import Logger, LoggerConfig

185
server/lib/python/cartodb_services/cartodb_services/tools/log.py Normal file
View File

@@ -0,0 +1,185 @@
import plpy
import rollbar
import logging
import json
import traceback
import sys
# Monkey patch because plpython sys module doesn't have argv and rollbar
# package use it
sys.__dict__['argv'] = []
class Logger:
LEVELS = {'debug': 1, 'info': 2, 'warning': 3, 'error': 4}
def __init__(self, config):
self._config = config
self._min_level = self.LEVELS[self._config.min_log_level]
# We need to set the handler blocking (synchronous) because
# spawn a thread from plpython interpreter don't work
if self._rollbar_activated():
rollbar.init(self._config.rollbar_api_key,
self._config.environment, handler='blocking')
if self._log_file_activated():
self._file_logger = self._setup_file_logger(
self._config.log_file_path)
def debug(self, text, exception=None, data={}):
if not self._check_min_level('debug'):
return
self._send_to_rollbar('debug', text, exception, data)
self._send_to_log_file('debug', text, exception, data)
plpy.debug(text)
def info(self, text, exception=None, data={}):
if not self._check_min_level('info'):
return
self._send_to_rollbar('info', text, exception, data)
self._send_to_log_file('info', text, exception, data)
plpy.info(text)
def warning(self, text, exception=None, data={}):
if not self._check_min_level('warning'):
return
self._send_to_rollbar('warning', text, exception, data)
self._send_to_log_file('warning', text, exception, data)
plpy.warning(text)
def error(self, text, exception=None, data={}):
if not self._check_min_level('error'):
return
self._send_to_rollbar('error', text, exception, data)
self._send_to_log_file('error', text, exception, data)
# Plpy.error and fatal raises exceptions and we only want to log an
# error, exceptions should be raise explicitly
plpy.warning(text)
def _check_min_level(self, level):
return True if self.LEVELS[level] >= self._min_level else False
def _send_to_rollbar(self, level, text, exception, data):
if self._rollbar_activated():
try:
if exception:
rollbar.report_exc_info(exception, extra_data=data,
level=level)
else:
rollbar.report_message(text, level, extra_data=data)
except Exception as e:
plpy.warning('Error sending message/exception to rollbar: {0}'.
format(e))
def _send_to_log_file(self, level, text, exception, data):
if self._log_file_activated():
extra_data = self._parse_log_extra_data(exception, data)
if level == 'debug':
self._file_logger.debug(text, extra=extra_data)
elif level == 'info':
self._file_logger.info(text, extra=extra_data)
elif level == 'warning':
self._file_logger.warning(text, extra=extra_data)
elif level == 'error':
self._file_logger.error(text, extra=extra_data)
def _parse_log_extra_data(self, exception, data):
extra_data = {}
if exception:
type_, value_, traceback_ = exception
exception_traceback = traceback.format_tb(traceback_)
extra_data = {"exception_type": type_, "exception_message": value_,
"exception_traceback": exception_traceback,
"log_data": data}
else:
extra_data = {"exception_type": '', "exception_message": '',
"exception_traceback": '', 'log_data': ''}
if data:
extra_data['data'] = data
else:
extra_data['data'] = ''
return extra_data
def _setup_file_logger(self, log_file_path):
logging.basicConfig(level='DEBUG')
formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s")
logger = logging.getLogger('dataservices_file_logger')
handler = logging.FileHandler(log_file_path)
handler.setFormatter(formatter)
handler.setLevel(self._config.min_log_level.upper())
logger.addHandler(handler)
return logger
def _rollbar_activated(self):
return True if self._config.rollbar_api_key else False
def _log_file_activated(self):
return True if self._config.log_file_path else False
class ConfigException(Exception):
pass
class LoggerConfig:
def __init__(self, db_conn):
self._db_conn = db_conn
return self._build()
def _build(self):
self._get_server_config()
self._get_logger_config()
def _get_server_config(self):
server_config_json = self._get_conf('server_conf')
if not server_config_json:
self._server_environment = 'development'
else:
server_config_json = json.loads(server_config_json)
if 'environment' in server_config_json:
self._server_environment = server_config_json['environment']
else:
self._server_environment = 'development'
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
logger_conf = json.loads(logger_conf_json)
self._rollbar_api_key = None
self._min_log_level = 'warning'
self._log_file_path = None
if 'min_log_level' in logger_conf:
self._min_log_level = logger_conf['min_log_level']
if 'rollbar_api_key' in logger_conf:
self._rollbar_api_key = logger_conf['rollbar_api_key']
if 'log_file_path' in logger_conf:
self._log_file_path = logger_conf['log_file_path']
def _get_conf(self, key):
try:
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
conf = self._db_conn.execute(sql, 1)
return conf[0]['conf']
except Exception as e:
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
@property
def environment(self):
return self._server_environment
@property
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property
def min_log_level(self):
return self._min_log_level

1
server/lib/python/cartodb_services/requirements.txt
View File

@@ -2,6 +2,7 @@ redis==2.10.5
hiredis==0.1.5
python-dateutil==2.2
googlemaps==2.4.2
rollbar==0.13.2
# Dependency for googlemaps package
requests<=2.9.1

2
server/lib/python/cartodb_services/setup.py
View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.7.2.3',
version='0.7.4',
description='CartoDB Services API Python Library',

4
server/lib/python/cartodb_services/test/test_googlegeocoder.py
View File

@@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
@@ -89,8 +90,9 @@ class GoogleGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
logger = Mock()
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E', logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,

4
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -1,5 +1,7 @@
from datetime import datetime, date
from mock import Mock
import sys
sys.modules['plpy'] = Mock()
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@@ -72,3 +74,5 @@ def _plpy_execute_side_effect(*args, **kwargs):
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
return [{'conf': '{"environment": "testing"}'}]

4
server/lib/python/cartodb_services/test/test_heremapsgeocoder.py
View File

@@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
@@ -102,7 +103,8 @@ class HereMapsGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = HereMapsGeocoder(None, None)
logger = Mock()
self.geocoder = HereMapsGeocoder(None, None, logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,

4
server/lib/python/cartodb_services/test/test_heremapsrouting.py
View File

@@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from urlparse import urlparse, parse_qs
from cartodb_services.here import HereMapsRoutingIsoline
@@ -127,7 +128,8 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.routing = HereMapsRoutingIsoline(None, None)
logger = Mock()
self.routing = HereMapsRoutingIsoline(None, None, logger)
self.isoline_url = "{0}{1}".format(HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL,
HereMapsRoutingIsoline.ISOLINE_PATH)

6
server/lib/python/cartodb_services/test/test_mapzengeocoder.py
View File

@@ -1,8 +1,9 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import mock
import unittest
import requests_mock
from mock import Mock
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.exceptions import MalformedResult
@@ -88,7 +89,8 @@ class MapzenGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = MapzenGeocoder('search-XXXXXXX')
logger = Mock()
self.geocoder = MapzenGeocoder('search-XXXXXXX', logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL,

4
server/lib/python/cartodb_services/test/test_mapzenisolines.py
View File

@@ -1,4 +1,5 @@
import unittest
from mock import Mock
from cartodb_services.mapzen import MapzenIsolines
from math import radians, cos, sin, asin, sqrt
@@ -10,6 +11,7 @@ It uses a mocked client, which returns the cost based on a very simple model:
just proportional to the distance from origin to the target point.
"""
class MatrixClientMock():
def __init__(self, speed):
@@ -67,7 +69,7 @@ class MapzenIsolinesTestCase(unittest.TestCase):
def setUp(self):
speed = 4 # in km/h
matrix_client = MatrixClientMock(speed)
self.mapzen_isolines = MapzenIsolines(matrix_client)
self.mapzen_isolines = MapzenIsolines(matrix_client, Mock())
def test_calculate_isochrone(self):
origin = {"lat":40.744014,"lon":-73.990508}

4
server/lib/python/cartodb_services/test/test_mapzenrouting.py
View File

@@ -6,6 +6,7 @@ import requests_mock
import re
from nose.tools import assert_raises
from urlparse import urlparse, parse_qs
from mock import Mock
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.exceptions import WrongParams
@@ -99,7 +100,8 @@ class MapzenRoutingTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.routing = MapzenRouting('api_key')
logger = Mock()
self.routing = MapzenRouting('api_key', logger)
self.url = MapzenRouting.PRODUCTION_ROUTING_BASE_URL
def test_calculate_simple_routing_with_valid_params(self, req_mock):