diff --git a/server/lib/python/cartodb_services/test/__init__.py b/server/lib/python/cartodb_services/test/__init__.py index e69de29..79a7695 100644 --- a/server/lib/python/cartodb_services/test/__init__.py +++ b/server/lib/python/cartodb_services/test/__init__.py @@ -0,0 +1,5 @@ +from test_helper import plpy_mock_config + + +def setup(): + plpy_mock_config() diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index a9142f0..8d0ceb5 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -1,13 +1,14 @@ from unittest import TestCase from mockredis import MockRedis +from ..test_helper import * from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig -from ..test_helper import build_plpy_mock + class TestRoutingConfig(TestCase): def setUp(self): self._redis_conn = MockRedis() - self._db_conn = build_plpy_mock() + self._db_conn = plpy_mock self._username = 'my_test_user' self._user_key = "rails:users:{0}".format(self._username) self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index c56756c..18f2105 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -1,6 +1,6 @@ from unittest import TestCase from mockredis import MockRedis -from ..test_helper import build_plpy_mock +from ..test_helper import * from cartodb_services.metrics.quota import QuotaChecker from cartodb_services.metrics import RoutingConfig from datetime import datetime diff --git a/server/lib/python/cartodb_services/test/mock_plpy.py b/server/lib/python/cartodb_services/test/mock_plpy.py new file mode 100644 index 0000000..5298a98 --- /dev/null +++ b/server/lib/python/cartodb_services/test/mock_plpy.py @@ -0,0 +1,53 @@ +import re + + +class MockCursor: + def __init__(self, data): + self.cursor_pos = 0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos: self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = {} + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results[pattern] = result + + def notice(self, msg): + self.notices.append(msg) + + def debug(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def cursor(self, query): + data = self.execute(query) + return MockCursor(data) + + def execute(self, query, rows=1): + for pattern, result in self.results.iteritems(): + if pattern.search(query): + return result + return [] diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 4a639ec..b55b991 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -1,20 +1,20 @@ -import test_helper -from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException +from test_helper import * from unittest import TestCase from nose.tools import assert_raises from mockredis import MockRedis from datetime import datetime, timedelta +from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() - self.plpy_mock = test_helper.build_plpy_mock() + plpy_mock_config() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', None) assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 @@ -22,10 +22,10 @@ class TestConfig(TestCase): def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - test_helper.build_redis_org_config(self.redis_conn, 'test_org', - quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + build_redis_org_config(self.redis_conn, 'test_org', + quota=200, end_date=yesterday) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', 'test_org') assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 @@ -34,10 +34,10 @@ class TestConfig(TestCase): def test_should_return_config_for_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - do_quota=100, soft_do_limit=True, - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + do_quota=100, soft_do_limit=True, + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 100 assert do_config.soft_limit is True @@ -45,16 +45,16 @@ class TestConfig(TestCase): def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 0 assert do_config.soft_limit is False assert do_config.period_end_date.date() == yesterday.date() def test_should_raise_exception_when_missing_parameters(self): - plpy_mock = test_helper.build_plpy_mock(empty=True) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') + plpy_mock._reset() + build_redis_user_config(self.redis_conn, 'test_user') assert_raises(ConfigException, GeocoderConfig, self.redis_conn, plpy_mock, 'test_user', None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index d6e9d80..2272278 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,7 +1,11 @@ from datetime import datetime, date -from mock import Mock +from mock import Mock, MagicMock +import random import sys -sys.modules['plpy'] = Mock() +from mock_plpy import MockPlPy + +plpy_mock = MockPlPy() +sys.modules['plpy'] = plpy_mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None, redis_conn.zincrby(redis_name, date.day, amount) -def build_plpy_mock(empty=False): - plpy_mock = Mock() - if not empty: - plpy_mock.execute.side_effect = _plpy_execute_side_effect - - return plpy_mock - - -def _plpy_execute_side_effect(*args, **kwargs): - if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": - return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": - return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": - return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf": - return [{'conf': '{"environment": "testing"}'}] +def plpy_mock_config(): + plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}]) + plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}]) diff --git a/server/lib/python/cartodb_services/test/test_metrics_log.py b/server/lib/python/cartodb_services/test/test_metrics_log.py new file mode 100644 index 0000000..963a34a --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_metrics_log.py @@ -0,0 +1,23 @@ +from test_helper import * +from cartodb_services.metrics import MetricsDataGatherer +from unittest import TestCase +from mock import Mock, MagicMock +from nose.tools import assert_raises +from datetime import datetime, date + + +class TestMetricsDataGatherer(TestCase): + + def setUp(self): + plpy_mock_config() + + + def test_should_use_multiple_instances_for_multiples_requests(self): + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + MetricsDataGatherer.add('test', 1) + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + MetricsDataGatherer.add('test', 2) + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + assert MetricsDataGatherer.get_element('test') is 1 + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + assert MetricsDataGatherer.get_element('test') is 2 diff --git a/server/lib/python/cartodb_services/test/test_qps.py b/server/lib/python/cartodb_services/test/test_qps.py index 014c510..bf396ca 100644 --- a/server/lib/python/cartodb_services/test/test_qps.py +++ b/server/lib/python/cartodb_services/test/test_qps.py @@ -1,4 +1,3 @@ -import test_helper import requests from unittest import TestCase from nose.tools import assert_raises diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 7cc49dd..c104235 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import QuotaService from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig @@ -28,32 +28,32 @@ class TestQuotaService(TestCase): def test_should_return_true_if_user_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user') + increment_service_uses(self.redis_conn, 'test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is True @@ -61,7 +61,7 @@ class TestQuotaService(TestCase): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is True @@ -140,18 +140,17 @@ class TestQuotaService(TestCase): def __prepare_quota_service(self, username, quota, service, orgname, soft_limit, do_quota, soft_do_limit, end_date): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, soft_do_limit=soft_do_limit, do_quota=do_quota, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, service=service, do_quota=do_quota, end_date=end_date) - self._plpy_mock = test_helper.build_plpy_mock() def __build_geocoder_quota_service(self, username, quota=100, service='heremaps', orgname=None, @@ -159,7 +158,7 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) @@ -168,7 +167,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, + routing_config = RoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) @@ -177,7 +176,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock, + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(isolines_config, redis_connection=self.redis_conn) @@ -188,6 +187,6 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, 0, service, orgname, False, quota, soft_limit, end_date) - do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock, + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index a73e00b..2905f8b 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import GeocoderConfig @@ -19,13 +19,13 @@ class TestUserService(TestCase): def test_user_used_quota_for_a_day(self): us = self.__build_user_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_org_used_quota_for_a_day(self): us = self.__build_user_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 @@ -135,14 +135,13 @@ class TestUserService(TestCase): def __build_user_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, end_date=date.today()): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - plpy_mock = test_helper.build_plpy_mock() geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return UserMetricsService(geocoder_config, self.redis_conn)