diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index 96786a7..168e6fd 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -58,8 +58,8 @@ RETURNS cdb_dataservices_server.simple_route AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen routing', sys.exc_info()) - raise e + logger.error('Error trying to calculate mapzen routing', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to calculate mapzen routing') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index f3d19cc..c8cfb61 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -61,8 +61,8 @@ RETURNS json AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to obst_get_demographic_snapshot', sys.exc_info()) - raise e + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -115,8 +115,8 @@ RETURNS SETOF JSON AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info()) - raise e + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -164,8 +164,8 @@ RETURNS json AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info()) - raise e + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -216,8 +216,8 @@ RETURNS SETOF JSON AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -270,8 +270,8 @@ RETURNS NUMERIC AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasure', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -322,8 +322,8 @@ RETURNS TEXT AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetCategory', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -376,8 +376,8 @@ RETURNS NUMERIC AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -428,8 +428,8 @@ RETURNS TEXT AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -480,8 +480,8 @@ RETURNS NUMERIC AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPopulation', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -532,8 +532,8 @@ RETURNS NUMERIC AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index c6d0863..8bfb46e 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -48,8 +48,8 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_Search', sys.exc_info()) - raise e + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -103,8 +103,8 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index f37d9a8..29f2b72 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -42,8 +42,8 @@ RETURNS geometry(Geometry, 4326) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundary', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -92,8 +92,8 @@ RETURNS TEXT AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -142,8 +142,8 @@ RETURNS geometry(Geometry, 4326) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -199,8 +199,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -258,8 +258,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -315,8 +315,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -374,8 +374,8 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info()) - raise e + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index bda170e..7ab9ca4 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -94,8 +94,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using here maps', sys.exc_info()) - raise e + logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using here maps') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -127,8 +127,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using google maps', sys.exc_info()) - raise e + logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using google maps') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -168,8 +168,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapzen', sys.exc_info()) - raise e + logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode street point using mapzen') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 1849344..0cdf472 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -25,8 +25,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin0 polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index d061fbd..67b766b 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -26,8 +26,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -60,8 +60,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 240f52e..d1afd1f 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -27,8 +27,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info()) - raise e + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -61,8 +61,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info()) - raise e + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -95,8 +95,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info()) - raise e + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 328d27c..bf158bd 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -25,8 +25,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info()) - raise e + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -58,8 +58,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info()) - raise e + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -91,8 +91,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; @@ -124,8 +124,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 4d519b5..fc37d3d 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -25,8 +25,8 @@ RETURNS Geometry AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info()) - raise e + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 17acbdf..79f5c6c 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -50,8 +50,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen isolines', sys.exc_info()) - raise e + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -123,8 +123,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ except BaseException as e: import sys quota_service.increment_failed_service_use() - logger.error('Error trying to get mapzen isolines', sys.exc_info()) - raise e + logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to get mapzen isolines') finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index a5f41b9..d39aa95 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -42,6 +42,7 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params) + raise Exception('testing exception handling') if not requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', data={"response": response.json(), "locations": diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 3c41cfe..4d22332 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -418,13 +418,13 @@ class ServicesDBConfig: def _get_server_config(self): server_config_json = self._get_conf('server_conf') if not server_config_json: - self._server_environment = 'production' + self._server_environment = 'development' else: server_config_json = json.loads(server_config_json) if 'environment' in server_config_json: self._server_environment = server_config_json['environment'] else: - self._server_environment = 'production' + self._server_environment = 'development' def _get_here_config(self): heremaps_conf_json = self._get_conf('heremaps_conf') diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index 822a8d1..f1df969 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -22,7 +22,8 @@ class Logger: rollbar.init(self._config.rollbar_api_key, self._config.environment, handler='blocking') if self._log_file_activated(): - self._file_logger = self._setup_file_logger(self._config.log_file_path) + self._file_logger = self._setup_file_logger( + self._config.log_file_path) def debug(self, text, exception=None, data={}): if not self._check_min_level('debug'): @@ -82,6 +83,7 @@ class Logger: self._file_logger.error(text, extra=extra_data) def _parse_log_extra_data(self, exception, data): + extra_data = {} if exception: type_, value_, traceback_ = exception exception_traceback = traceback.format_tb(traceback_) @@ -90,7 +92,7 @@ class Logger: "log_data": data} else: extra_data = {"exception_type": '', "exception_message": '', - "exception_traceback": ''} + "exception_traceback": '', 'log_data': ''} if data: extra_data['data'] = data @@ -100,10 +102,11 @@ class Logger: return extra_data def _setup_file_logger(self, log_file_path): - format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s" + logging.basicConfig(level='DEBUG') + formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s") logger = logging.getLogger('dataservices_file_logger') handler = logging.FileHandler(log_file_path) - handler.setFormatter(format_str) + handler.setFormatter(formatter) handler.setLevel(self._config.min_log_level.upper()) logger.addHandler(handler)