diff --git a/README.md b/README.md index b0363d5..0f52b1c 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,11 @@ Steps to deploy a new Data Services API version : SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}'); SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}'); - SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}') + SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}') SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}') + + # Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior) + SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}') ``` - configure the user DB: diff --git a/server/extension/Makefile b/server/extension/Makefile index dcce17f..bb98dc2 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -10,7 +10,7 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) TEST_DIR = test/ -REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres' # DATA is a special variable used by postgres build infrastructure # These are the files to be installed in the server shared dir, diff --git a/server/extension/cdb_dataservices_server--0.13.3.1--0.14.sql b/server/extension/cdb_dataservices_server--0.13.3.1--0.14.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server--0.14--0.13.3.1.sql b/server/extension/cdb_dataservices_server--0.14--0.13.3.1.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index cccdc99..ee2149c 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.13.3.1' +default_version = '0.14' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.13.3--0.13.3.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.3--0.13.3.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.3--0.13.3.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.3--0.13.3.1.sql diff --git a/server/extension/cdb_dataservices_server--0.13.3.1--0.13.3.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.3.1--0.13.3.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.3.1--0.13.3.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.3.1--0.13.3.sql diff --git a/server/extension/cdb_dataservices_server--0.13.3.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.13.3.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.13.3.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.13.3.1.sql diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index 30ab58f..cf72a90 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -17,19 +17,22 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.mapzen.types import polyline_to_linestring from cartodb_services.metrics import QuotaService from cartodb_services.tools import Coordinate + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_routing_config = GD["user_routing_config_{0}".format(username)] + logger = Logger(user_routing_config) + quota_service = QuotaService(user_routing_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: - client = MapzenRouting(user_routing_config.mapzen_api_key) + client = MapzenRouting(user_routing_config.mapzen_api_key, logger) if not waypoints or len(waypoints) < 2: - plpy.notice("Empty origin or destination") + logger.notice("Empty origin or destination") quota_service.increment_empty_service_use() return [None, None, None] @@ -52,12 +55,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$ quota_service.increment_empty_service_use() return [None, None, None] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to get mapzen routing', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index 7792c5a..f00b6f0 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -35,6 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -42,9 +43,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) @@ -56,12 +58,10 @@ RETURNS json AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to obst_get_demographic_snapshot', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -85,15 +85,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) @@ -109,12 +111,10 @@ RETURNS SETOF JSON AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -136,6 +136,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -143,9 +144,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) @@ -157,12 +159,10 @@ RETURNS json AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -184,15 +184,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) @@ -208,12 +210,10 @@ RETURNS SETOF JSON AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -241,15 +241,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) @@ -261,12 +263,10 @@ RETURNS NUMERIC AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetMeasure', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -292,15 +292,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) @@ -312,12 +314,10 @@ RETURNS TEXT AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetCategory', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -345,15 +345,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) @@ -365,12 +367,10 @@ RETURNS NUMERIC AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -396,15 +396,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) @@ -416,12 +418,10 @@ RETURNS TEXT AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -447,15 +447,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) @@ -467,12 +469,10 @@ RETURNS NUMERIC AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetPopulation', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -498,15 +498,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) @@ -518,12 +520,10 @@ RETURNS NUMERIC AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index fbf38cc..42be50f 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -15,15 +15,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( relevant_boundary TEXT DEFAULT NULL) RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) @@ -43,12 +45,10 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) quota_service.increment_empty_service_use() return [None, None, None, None, None] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_Search', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -70,15 +70,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( time_span TEXT DEFAULT NULL) RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) @@ -97,12 +99,10 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index 4281220..110b481 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -17,15 +17,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) @@ -37,12 +39,10 @@ RETURNS geometry(Geometry, 4326) AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetBoundary', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -66,15 +66,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) @@ -86,12 +88,10 @@ RETURNS TEXT AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use obs_search: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -115,15 +115,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) @@ -135,12 +137,10 @@ RETURNS geometry(Geometry, 4326) AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -166,15 +166,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) @@ -191,12 +193,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -224,15 +224,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) @@ -249,12 +251,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -280,15 +280,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) @@ -305,12 +307,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -338,15 +338,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] + logger = Logger(user_obs_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) @@ -363,12 +365,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 749ddee..edd6056 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -7,19 +7,16 @@ RETURNS Geometry AS $$ user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] if user_geocoder_config.heremaps_geocoder: - plpy.debug('Requested geocoder is heremaps') here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] elif user_geocoder_config.google_geocoder: - plpy.debug('Requested geocoder is google') google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] elif user_geocoder_config.mapzen_geocoder: - plpy.debug('Requested geocoder is mapzen') mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: - plpy.error('Requested geocoder is not available') + raise Exception('Requested geocoder is not available') $$ LANGUAGE plpythonu; @@ -35,7 +32,7 @@ RETURNS Geometry AS $$ here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: - plpy.error('Here geocoder is not available for your account.') + raise Exception('Here geocoder is not available for your account.') $$ LANGUAGE plpythonu; @@ -50,7 +47,7 @@ RETURNS Geometry AS $$ google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: - plpy.error('Google geocoder is not available for your account.') + raise Exception('Google geocoder is not available for your account.') $$ LANGUAGE plpythonu; @@ -71,17 +68,19 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) # -- Check the quota quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: - geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -92,12 +91,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode street point using here maps', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -106,13 +103,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po RETURNS Geometry AS $$ from cartodb_services.google import GoogleMapsGeocoder from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: - geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -123,12 +123,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode street point using google maps', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -138,15 +136,18 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key) + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) country_iso3 = None if country: country_iso3 = country_to_iso3(country) @@ -162,12 +163,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode street point using mapzen', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index c6c0346..93b9bea 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) @@ -20,12 +22,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode admin0 polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index f75d633..208875a 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -3,12 +3,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) @@ -21,12 +23,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode admin1 polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -36,12 +36,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) @@ -54,12 +56,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode admin1 polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 2a4676a..070b2fd 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -3,12 +3,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) @@ -21,12 +24,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode namedplace point', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -36,12 +37,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) @@ -54,12 +57,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode namedplace point', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -69,12 +70,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) @@ -87,12 +90,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode namedplace point', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 14f5da6..6dd4ffd 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) @@ -20,12 +22,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode postal code point', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -34,12 +34,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) @@ -52,12 +54,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode postal code point', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -66,12 +66,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) @@ -84,12 +86,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode postal code polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -98,12 +98,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) @@ -116,12 +118,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode postal code polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index cca19a8..2cc9259 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(u RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + logger = Logger(user_geocoder_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) @@ -20,12 +22,10 @@ RETURNS Geometry AS $$ quota_service.increment_empty_service_use() return None except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to geocode postal code polygon', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 597eebd..a3deb7b 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -6,17 +6,20 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.here import HereMapsRoutingIsoline from cartodb_services.metrics import QuotaService from cartodb_services.here.types import geo_polyline_to_multipolygon + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + logger = Logger(user_isolines_routing_config) # -- Check the quota quota_service = QuotaService(user_isolines_routing_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: - client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + user_isolines_routing_config.heremaps_app_code, logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -44,12 +47,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ quota_service.increment_empty_service_use() return [] except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) + logger.error('Error trying to get mapzen isolines', sys.exc_info()) + raise e finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -65,21 +66,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines( options text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ import json - from cartodb_services.mapzen import MatrixClient - from cartodb_services.mapzen import MapzenIsolines + from cartodb_services.mapzen import MatrixClient, MapzenIsolines from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Logger redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] + logger = Logger(user_isolines_routing_config) # -- Check the quota quota_service = QuotaService(user_isolines_routing_config, redis_conn) if not quota_service.check_user_quota(): - plpy.error('You have reached the limit of your quota') + raise Exception('You have reached the limit of your quota') try: - client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key) - mapzen_isolines = MapzenIsolines(client) + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger) + mapzen_isolines = MapzenIsolines(client, logger) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -117,13 +119,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ quota_service.increment_isolines_service_use(len(isolines)) return result except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() + import sys quota_service.increment_failed_service_use() - error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e) - plpy.debug(traceback.format_tb(traceback_)) + logger.error('Error trying to get mapzen isolines', sys.exc_info()) raise e - #plpy.error(error_msg) finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/85_isodistance.sql b/server/extension/sql/85_isodistance.sql index a87a9f7..874ae29 100644 --- a/server/extension/sql/85_isodistance.sql +++ b/server/extension/sql/85_isodistance.sql @@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] if user_isolines_config.google_services_user: - plpy.error('This service is not available for google service users.') + raise Exception('This service is not available for google service users.') if user_isolines_config.heremaps_provider: - plpy.debug('Requested isolines provider is heremaps') here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - plpy.debug('Requested isolines provider is mapzen') mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) else: - plpy.error('Requested isolines provider is not available') + raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu; -- heremaps isodistance diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 9aee838..24d336e 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] if user_isolines_config.google_services_user: - plpy.error('This service is not available for google service users.') + raise Exception('This service is not available for google service users.') if user_isolines_config.heremaps_provider: - plpy.debug('Requested isolines provider is heremaps') here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) elif user_isolines_config.mapzen_provider: - plpy.debug('Requested isolines provider is mapzen') mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) else: - plpy.error('Requested isolines provider is not available') + raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu; -- heremaps isochrone diff --git a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py index ed2a701..2a836f3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/google/geocoder.py @@ -9,11 +9,12 @@ from exceptions import MalformedResult class GoogleMapsGeocoder: """A Google Maps Geocoder wrapper for python""" - def __init__(self, client_id, client_secret): + def __init__(self, client_id, client_secret, logger): self.client_id = self._clean_client_id(client_id) self.client_secret = client_secret self.geocoder = googlemaps.Client( client_id=self.client_id, client_secret=self.client_secret) + self._logger = logger def geocode(self, searchtext, city=None, state=None, country=None): diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 8e93316..1ae6ca8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -47,10 +47,11 @@ class HereMapsGeocoder: 'strictlanguagemode' ] + ADDRESS_PARAMS - def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, + def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS, gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): self.app_id = app_id self.app_code = app_code + self._logger = logger self.maxresults = maxresults self.gen = gen self.host = host @@ -88,9 +89,15 @@ class HereMapsGeocoder: if response.status_code == requests.codes.ok: return json.loads(response.text) elif response.status_code == requests.codes.bad_request: + self._logger.warning('Error 4xx trying to geocode street using HERE', + data={"response": response.json(), "params": + params}) return [] else: - response.raise_for_status() + self._logger.error('Error trying to geocode street using HERE', + data={"response": response.json(), "params": + params}) + raise Exception('Error trying to geocode street using Here') def _extract_lng_lat_from_result(self, result): location = result['Location'] diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 71d9cc9..5c01f84 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -25,9 +25,11 @@ class HereMapsRoutingIsoline: 'quality' ] - def __init__(self, app_id, app_code, base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_id, app_code, logger, + base_url=PRODUCTION_ROUTING_BASE_URL): self._app_id = app_id self._app_code = app_code + self._logger = logger self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH) def calculate_isodistance(self, source, mode, data_range, options=[]): @@ -54,7 +56,12 @@ class HereMapsRoutingIsoline: elif response.status_code == requests.codes.bad_request: return [] else: - response.raise_for_status() + self._logger.error('Error trying to calculate HERE isolines', + data={"response": response.json(), "source": + source, "mode": mode, "data_range": + data_range, "range_type": range_type, + "options": options}) + raise Exception('Error trying to calculate HERE isolines') def __parse_options(self, options): return dict(option.split('=') for option in options) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 9adbfea..5816950 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -12,9 +12,10 @@ class MapzenGeocoder: BASE_URL = 'https://search.mapzen.com/v1/search' - def __init__(self, app_key, base_url=BASE_URL): + def __init__(self, app_key, logger, base_url=BASE_URL): self._app_key = app_key self._url = base_url + self._logger = logger @qps_retry def geocode(self, searchtext, city=None, state_province=None, country=None): @@ -25,7 +26,7 @@ class MapzenGeocoder: elif response.status_code == requests.codes.bad_request: return [] else: - response.raise_for_status() + raise Exception('Error trying to geocode {0} using mapzen'.format(searchtext)) def _build_requests_parameters(self, searchtext, city=None, state_province=None, country=None): diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index e73d8e7..a5f41b9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -2,6 +2,7 @@ import requests import json from qps import qps_retry + class MatrixClient: """ @@ -42,8 +43,14 @@ class MatrixClient: response = requests.get(self.ONE_TO_MANY_URL, params=request_params) if not requests.codes.ok: - self._logger.warning('Error trying to get matrix distance from mapzen', data={"response": response, "locations": locations, "costing": costing}) - - response.raise_for_status() # raise exception if not 200 OK + self._logger.error('Error trying to get matrix distance from mapzen', + data={"response": response.json(), "locations": + locations, "costing": costing}) + raise Exception('Error trying to get matrix distance from mapzen') + else: + self._logger.debug('Done get matrix distance from mapzen', + data={"response": response.json(), + "locations": locations, + "costing": costing}) return response.json() diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index 80547bb..72a9841 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -28,9 +28,10 @@ class MapzenRouting: METRICS_UNITS = 'kilometers' IMPERIAL_UNITS = 'miles' - def __init__(self, app_key, base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL): self._app_key = app_key self._url = base_url + self._logger = logger @qps_retry def calculate_route_point_to_point(self, waypoints, mode, @@ -48,7 +49,11 @@ class MapzenRouting: elif response.status_code == requests.codes.bad_request: return MapzenRoutingResponse(None, None, None) else: - response.raise_for_status() + self._logger.error('Error trying to calculate route using HERE', + data={"response": response.json(), "waypoints": + waypoints, "mode": mode, "options": + options}) + raise Exception('Error trying to calculate route using HERE') def __parse_options(self, options): return dict(option.split('=') for option in options) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 70322af..33dbb37 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -17,6 +17,8 @@ class ServiceConfig(object): self._db_config = ServicesDBConfig(db_conn, username, orgname) self._environment = self._db_config._server_environment self._rollbar_api_key = self._db_config._rollbar_api_key + self._log_file_path = self._db_config._log_file_path + self._min_log_level = self._db_config._min_log_level if redis_connection: self._redis_config = ServicesRedisConfig(redis_connection).build( username, orgname) @@ -39,10 +41,19 @@ class ServiceConfig(object): def rollbar_api_key(self): return self._rollbar_api_key + @property + def log_file_path(self): + return self._log_file_path + + @property + def min_log_level(self): + return self._min_log_level + @property def environment(self): return self._environment + class DataObservatoryConfig(ServiceConfig): def __init__(self, redis_connection, db_conn, username, orgname=None): @@ -65,6 +76,7 @@ class DataObservatoryConfig(ServiceConfig): def connection_str(self): return self._connection_str + class ObservatorySnapshotConfig(DataObservatoryConfig): SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit' @@ -88,6 +100,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig): def service_type(self): return 'obs_snapshot' + class ObservatoryConfig(DataObservatoryConfig): SOFT_LIMIT_KEY = 'soft_obs_general_limit' @@ -111,6 +124,7 @@ class ObservatoryConfig(DataObservatoryConfig): def service_type(self): return 'obs_general' + class RoutingConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' @@ -172,6 +186,7 @@ class IsolinesRoutingConfig(ServiceConfig): if not self._isolines_provider: self._isolines_provider = self.DEFAULT_PROVIDER self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower() + self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if self._isolines_provider == self.HEREMAPS_PROVIDER: self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) self._heremaps_app_id = db_config.heremaps_isolines_app_id @@ -184,7 +199,6 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota self._soft_isolines_limit = False - self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) @property def service_type(self): @@ -473,10 +487,15 @@ class ServicesDBConfig: else: logger_conf = json.loads(logger_conf_json) self._geocoder_log_path = logger_conf['geocoder_log_path'] + self._rollbar_api_key = None + self._min_log_level = 'warning' + self._log_file_path = None + if 'min_log_level' in logger_conf: + self._min_log_level = logger_conf['min_log_level'] if 'rollbar_api_key' in logger_conf: self._rollbar_api_key = logger_conf['rollbar_api_key'] - else: - self._rollbar_api_key = None + if 'log_file_path' in logger_conf: + self._log_file_path = logger_conf['log_file_path'] def _get_conf(self, key): try: @@ -542,6 +561,10 @@ class ServicesDBConfig: def rollbar_api_key(self): return self._rollbar_api_key + @property + def log_file_path(self): + return self._log_file_path + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index fdb533f..55a007e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,4 +1,4 @@ from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine -from log import Logger +from log import Logger \ No newline at end of file diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index ae98e41..dbd49ac 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -1,5 +1,7 @@ import plpy import rollbar +import logging +import traceback import sys # Monkey path because plpython sys module doesn't have argv and rollbar # package use it @@ -8,28 +10,51 @@ sys.__dict__['argv'] = [] class Logger: + LEVELS = {'debug': 1, 'info': 2, 'warning': 3, 'error': 4} + def __init__(self, config): self._config = config + self._min_level = self.LEVELS[self._config.min_log_level] # We need to set the handler blocking (synchronous) because # spawn a thread from plpython interpreter don't work - rollbar.init(self._config.rollbar_api_key, - self._config.environment, handler='blocking') + if self._rollbar_activated(): + rollbar.init(self._config.rollbar_api_key, + self._config.environment, handler='blocking') + if self._log_file_activated(): + self._setup_log_file_config(self._config.log_file_path) def debug(self, text, exception=None, data={}): + if not self._check_min_level('debug'): + return self._send_to_rollbar('debug', text, exception, data) + self._send_to_log_file('debug', text, exception, data) plpy.debug(text) def info(self, text, exception=None, data={}): + if not self._check_min_level('info'): + return self._send_to_rollbar('info', text, exception, data) + self._send_to_log_file('info', text, exception, data) plpy.info(text) def warning(self, text, exception=None, data={}): + if not self._check_min_level('warning'): + return self._send_to_rollbar('warning', text, exception, data) + self._send_to_log_file('warning', text, exception, data) plpy.warning(text) def error(self, text, exception=None, data={}): + if not self._check_min_level('error'): + return self._send_to_rollbar('error', text, exception, data) - plpy.error(text) + self._send_to_log_file('error', text, exception, data) + # Plpy.error and fatal raises exceptions and we only want to log an + # error, exceptions should be raise explicitly + plpy.warning(text) + + def _check_min_level(self, level): + return True if self.LEVELS[level] >= self._min_level else False def _send_to_rollbar(self, level, text, exception, data): if self._rollbar_activated(): @@ -43,5 +68,41 @@ class Logger: plpy.warning('Error sending message/exception to rollbar: {0}'. format(e)) + def _send_to_log_file(self, level, text, exception, data): + extra_data = self._parse_log_extra_data(exception, data) + if level == 'debug': + logging.debug(text, extra=extra_data) + elif level == 'info': + logging.info(text, extra=extra_data) + elif level == 'warning': + logging.warning(text, extra=extra_data) + elif level == 'error': + logging.error(text, extra=extra_data) + + def _parse_log_extra_data(self, exception, data): + if exception: + type_, value_, traceback_ = exception + exception_traceback = traceback.format_tb(traceback_) + extra_data = {"exception_type": type_, "exception_message": value_, + "exception_traceback": exception_traceback, + "log_data": data} + else: + extra_data = {"exception_type": '', "exception_message": '', + "exception_traceback": ''} + + if data: + extra_data['data'] = data + else: + extra_data['data'] = '' + + return extra_data + + def _setup_log_file_config(self, log_file_path): + format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s" + logging.basicConfig(filename=log_file_path, format=format_str, level=self._config.min_log_level.upper()) + def _rollbar_activated(self): return True if self._config.rollbar_api_key else False + + def _log_file_activated(self): + return True if self._config.log_file_path else False diff --git a/server/lib/python/cartodb_services/test/test_googlegeocoder.py b/server/lib/python/cartodb_services/test/test_googlegeocoder.py index a015272..5593560 100644 --- a/server/lib/python/cartodb_services/test/test_googlegeocoder.py +++ b/server/lib/python/cartodb_services/test/test_googlegeocoder.py @@ -3,6 +3,7 @@ import unittest import requests_mock +from mock import Mock from cartodb_services.google import GoogleMapsGeocoder from cartodb_services.google.exceptions import BadGeocodingParams @@ -89,8 +90,9 @@ class GoogleGeocoderTestCase(unittest.TestCase): MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): + logger = Mock() self.geocoder = GoogleMapsGeocoder('dummy_client_id', - 'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E') + 'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E', logger) def test_geocode_address_with_valid_params(self, req_mock): req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL, diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 06bd2d2..d6e9d80 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,5 +1,7 @@ from datetime import datetime, date from mock import Mock +import sys +sys.modules['plpy'] = Mock() def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -72,3 +74,5 @@ def _plpy_execute_side_effect(*args, **kwargs): return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}] + elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf": + return [{'conf': '{"environment": "testing"}'}] diff --git a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py index c6cd790..c09d75b 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py @@ -3,6 +3,7 @@ import unittest import requests_mock +from mock import Mock from cartodb_services.here import HereMapsGeocoder from cartodb_services.here.exceptions import BadGeocodingParams @@ -102,7 +103,8 @@ class HereMapsGeocoderTestCase(unittest.TestCase): MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): - self.geocoder = HereMapsGeocoder(None, None) + logger = Mock() + self.geocoder = HereMapsGeocoder(None, None, logger) def test_geocode_address_with_valid_params(self, req_mock): req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, diff --git a/server/lib/python/cartodb_services/test/test_heremapsrouting.py b/server/lib/python/cartodb_services/test/test_heremapsrouting.py index a6ce503..ec860bf 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsrouting.py +++ b/server/lib/python/cartodb_services/test/test_heremapsrouting.py @@ -3,6 +3,7 @@ import unittest import requests_mock +from mock import Mock from urlparse import urlparse, parse_qs from cartodb_services.here import HereMapsRoutingIsoline @@ -127,7 +128,8 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase): MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): - self.routing = HereMapsRoutingIsoline(None, None) + logger = Mock() + self.routing = HereMapsRoutingIsoline(None, None, logger) self.isoline_url = "{0}{1}".format(HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL, HereMapsRoutingIsoline.ISOLINE_PATH) diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py index 7eee6c0..3a4aff5 100644 --- a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -1,8 +1,9 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- - +import mock import unittest import requests_mock +from mock import Mock from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.exceptions import MalformedResult @@ -88,7 +89,8 @@ class MapzenGeocoderTestCase(unittest.TestCase): MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): - self.geocoder = MapzenGeocoder('search-XXXXXXX') + logger = Mock() + self.geocoder = MapzenGeocoder('search-XXXXXXX', logger) def test_geocode_address_with_valid_params(self, req_mock): req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL, diff --git a/server/lib/python/cartodb_services/test/test_mapzenisolines.py b/server/lib/python/cartodb_services/test/test_mapzenisolines.py index 4fb4e96..cfb8f86 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenisolines.py +++ b/server/lib/python/cartodb_services/test/test_mapzenisolines.py @@ -1,4 +1,5 @@ import unittest +from mock import Mock from cartodb_services.mapzen import MapzenIsolines from math import radians, cos, sin, asin, sqrt @@ -10,6 +11,7 @@ It uses a mocked client, which returns the cost based on a very simple model: just proportional to the distance from origin to the target point. """ + class MatrixClientMock(): def __init__(self, speed): @@ -67,7 +69,7 @@ class MapzenIsolinesTestCase(unittest.TestCase): def setUp(self): speed = 4 # in km/h matrix_client = MatrixClientMock(speed) - self.mapzen_isolines = MapzenIsolines(matrix_client) + self.mapzen_isolines = MapzenIsolines(matrix_client, Mock()) def test_calculate_isochrone(self): origin = {"lat":40.744014,"lon":-73.990508} diff --git a/server/lib/python/cartodb_services/test/test_mapzenrouting.py b/server/lib/python/cartodb_services/test/test_mapzenrouting.py index 483db1d..2315146 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapzenrouting.py @@ -6,6 +6,7 @@ import requests_mock import re from nose.tools import assert_raises from urlparse import urlparse, parse_qs +from mock import Mock from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse from cartodb_services.mapzen.exceptions import WrongParams @@ -99,7 +100,8 @@ class MapzenRoutingTestCase(unittest.TestCase): MALFORMED_RESPONSE = """{"manolo": "escobar"}""" def setUp(self): - self.routing = MapzenRouting('api_key') + logger = Mock() + self.routing = MapzenRouting('api_key', logger) self.url = MapzenRouting.PRODUCTION_ROUTING_BASE_URL def test_calculate_simple_routing_with_valid_params(self, req_mock):