cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cartodb geocoder python module tests fixed

Browse Source
This commit is contained in:
Mario de Frutos
2016-01-21 18:07:27 +01:00
parent aced68096b
commit c74947b0a5
6 changed files with 237 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py
View File

@@ -23,9 +23,6 @@ class QuotaService:
current_used = self._user_service.used_quota(service_type, today)
soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit
print "User quota: {0} --- current_used: {1} --- limit: {2}".format(
user_quota, current_used, soft_geocoding_limit)
if soft_geocoding_limit or current_used <= user_quota:
return True
else:

38
server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py
View File

@@ -27,17 +27,17 @@ class UserService:
""" Recover the used quota for the user in the current month """
date_from, date_to = self.__current_billing_cycle()
current_use = 0
success_responses = self.__get_metrics(service_type,
'success_responses', date_from,
date_to)
empty_responses = self.__get_metrics(service_type,
'empty_responses', date_from,
success_responses = self.get_metrics(service_type,
'success_responses', date_from,
date_to)
empty_responses = self.get_metrics(service_type,
'empty_responses', date_from,
date_to)
current_use += (success_responses + empty_responses)
if service_type == self.SERVICE_GEOCODER_NOKIA:
cache_hits = self.__get_metrics(self.SERVICE_GEOCODER_CACHE,
'success_responses', date_from,
date_to)
cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE,
'success_responses', date_from,
date_to)
current_use += cache_hits
return current_use
@@ -48,6 +48,17 @@ class UserService:
if self._orgname:
self.__increment_organization_uses(service_type, metric, date, amount)
def get_metrics(self, service, metric, date_from, date_to):
aggregated_metric = 0
key_prefix = "org" if self._orgname else "user"
entity_name = self._orgname if self._orgname else self._username
for date in self.__generate_date_range(date_from, date_to):
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
service, metric, date)
score = self._redis_connection.zscore(redis_prefix, date.day)
aggregated_metric += score if score else 0
return aggregated_metric
# Private functions
def __increment_user_uses(self, service_type, metric, date, amount):
@@ -68,17 +79,6 @@ class UserService:
return redis_name
def __get_metrics(self, service, metric, date_from, date_to):
aggregated_metric = 0
key_prefix = "org" if self._orgname else "user"
entity_name = self._orgname if self._orgname else self._username
for date in self.__generate_date_range(date_from, date_to):
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
service, metric, date)
score = self._redis_connection.zscore(redis_prefix, date.day)
aggregated_metric += score if score else 0
return aggregated_metric
def __current_billing_cycle(self):
""" Return the begining and end date for the current billing cycle """
end_period_day = self._user_geocoder_config.period_end_date.day

69
server/lib/python/cartodb_geocoder/test/test_config_helper.py
View File

@@ -1,48 +1,41 @@
import test_helper
from cartodb_geocoder import config_helper
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfigHelper(TestCase):
def test_should_return_list_of_user_config_if_its_ok(self):
user_config_json = '{"is_organization": false, "entity_name": "test_user"}'
user_config = config_helper.UserConfig(user_config_json, 'development_cartodb_user_UUID')
assert user_config.is_organization == False
assert user_config.entity_name == 'test_user'
assert user_config.user_id == 'UUID'
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_raise_config_exception_if_not_ok(self):
user_config_json = '{"is_organization": "false"}'
assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json)
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit == False
def test_should_return_raise_config_exception_if_empty(self):
user_config_json = '{}'
assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json)
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
geocoder_config_json = """{"street_geocoder_provider": "Nokia",
"nokia_monthly_quota": 100, "nokia_soft_geocoder_limit": false}"""
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
assert geocoder_config.nokia_geocoder == True
assert geocoder_config.nokia_monthly_quota == 100
assert geocoder_config.nokia_soft_limit == False
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
geocoder_config_json = '{"street_geocoder_provider": "NokiA", "nokia_monthly_quota": "100"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters_2(self):
geocoder_config_json = '{"street_geocoder_provider": "NoKia", "nokia_soft_geocoder_limit": "false"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_return_list_of_google_geocoder_config_if_its_ok(self):
geocoder_config_json = """{"street_geocoder_provider": "gOOgle",
"google_maps_private_key": "sdasdasda"}"""
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
assert geocoder_config.google_geocoder == True
assert geocoder_config.google_api_key == 'sdasdasda'
def test_should_raise_configuration_exception_when_missing_google_api_key(self):
geocoder_config_json = '{"street_geocoder_provider": "google"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(config_helper.ConfigException,
config_helper.GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

33
server/lib/python/cartodb_geocoder/test/test_helper.py Normal file
View File

@@ -0,0 +1,33 @@
from datetime import datetime, date
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
service="heremaps",
end_date=datetime.today()):
user_redis_name = "rails:users:{0}".format(username)
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
def build_redis_org_config(redis_conn, orgname, quota=100,
end_date=datetime.today()):
org_redis_name = "rails:orgs:{0}".format(orgname)
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
def increment_geocoder_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
prefix = 'org' if orgname else 'user'
entity_name = orgname if orgname else username
yearmonth = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service, metric, yearmonth)
redis_conn.zincrby(redis_name, date.day, amount)

148
server/lib/python/cartodb_geocoder/test/test_quota_service.py
View File

@@ -1,85 +1,109 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import quota_service
from cartodb_geocoder import config_helper
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime
from datetime import datetime, date
class TestQuotaService(TestCase):
# single user
# user:<username>:<service>:used_quota_month:year_month
# user:<username>:<service>:used_quota_day:year_month_day
# organization user
# org:<orgname>:<service>:used_quota_month:year_month
# org:<orgname>:<service>:<uuid>:used_quota_day:year_month_day
# single user
# user:<username>:<service>:<metric>:YYYYMM:DD
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
def setUp(self):
self.fake_redis_connection = MockRedis()
# def increment_geocoder_uses(self, username, orgname=None,
# date=date.today(), service='geocoder_here',
# metric='success_responses', amount=20):
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service()
assert qs.check_user_quota() == True
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service(organization=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service('test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service()
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == True
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service(organization=True)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service(quota = 1, soft_limit=False)
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == False
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service(organization=True, quota=1)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == False
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service(quota=1, soft_limit=True)
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == True
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service(organization=True, quota=1, soft_limit=True)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service(quota=2, soft_limit=False)
qs.increment_geocoder_use()
assert qs.check_user_quota() == True
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service(organization=True, quota=2, soft_limit=False)
qs.increment_geocoder_use()
assert qs.check_user_quota() == True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
month = date.today().strftime('%Y%m')
name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
total_requests = self.redis_conn.zscore(name, date.today().day)
assert total_requests == 3
def __build_quota_service(self, quota=100, service='nokia', organization=False, soft_limit=False):
is_organization = 'true' if organization else 'false'
has_soft_limit = 'true' if soft_limit else 'false'
user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization)
geocoder_config_json = """{{"street_geocoder_provider": "{0}","nokia_monthly_quota": {1},
"nokia_soft_geocoder_limit": {2}}}""".format(service, quota, has_soft_limit)
user_config = config_helper.UserConfig(user_config_json, 'user_id')
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
month = date.today().strftime('%Y%m')
org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month)
org_total_requests = self.redis_conn.zscore(org_name, date.today().day)
assert org_total_requests == 3
user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
user_total_requests = self.redis_conn.zscore(user_name, date.today().day)
assert user_total_requests == 3
return quota_service.QuotaService(user_config, geocoder_config, redis_connection = self.fake_redis_connection)
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date = datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota = quota, service = service,
soft_limit = soft_limit,
end_date = end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return quota_service.QuotaService(geocoder_config,
redis_connection = self.redis_conn)
def __increment_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False):
prefix = 'org' if org else 'user'
date = datetime.strptime(date_string, "%Y%m%d")
redis_name = "{0}:{1}:{2}:used_quota_month".format(prefix, entity_name, service)
redis_key_month = "{0}_{1}".format(date.year, date.month)
self.fake_redis_connection.hset(redis_name, redis_key_month, amount)

109
server/lib/python/cartodb_geocoder/test/test_user_service.py
View File

@@ -1,63 +1,90 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import user_service
from cartodb_geocoder import config_helper
from datetime import datetime
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
from datetime import timedelta
class TestUserService(TestCase):
NOKIA_GEOCODER = 'nokia'
NOKIA_GEOCODER = 'geocoder_here'
def setUp(self):
self.fake_redis_connection = MockRedis()
self.redis_conn = MockRedis()
def test_user_used_quota_for_a_month(self):
us = self.__build_user_service()
self.__increment_month_geocoder_uses('test_user', '20151111')
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_month(self):
us = self.__build_user_service(organization=True)
self.__increment_month_geocoder_uses('test_user', '20151111', org=True)
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service()
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0
us = self.__build_user_service('test_user')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_org_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service(organization=True)
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0
us = self.__build_user_service('test_user', orgname='test_org')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_should_increment_user_used_quota(self):
us = self.__build_user_service()
date = datetime.strptime("20151111", "%Y%m%d")
us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1)
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1
def test_should_increment_user_used_quota_for_one_date(self):
us = self.__build_user_service('test_user')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_should_increment_org_used_quota(self):
us = self.__build_user_service(organization=True)
date = datetime.strptime("20151111", "%Y%m%d")
us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1)
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1
assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1
us = self.__build_user_service('test_user', orgname='test_org')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_exception_if_not_redis_config(self):
assert_raises(Exception, user_service.UserService, 'user_id')
def test_should_increment_user_used_quota_in_for_multiples_dates(self):
two_days_ago = datetime.today() - timedelta(days=2)
one_day_ago = datetime.today() - timedelta(days=1)
one_day_after = datetime.today() + timedelta(days=1)
us = self.__build_user_service('test_user', end_date=one_day_ago)
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
date=date.today())
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=two_days_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_after)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def __build_user_service(self, organization=False, service='nokia'):
is_organization = 'true' if organization else 'false'
user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization)
user_config = config_helper.UserConfig(user_config_json, 'user_id')
return user_service.UserService(user_config, service, redis_connection = self.fake_redis_connection)
def __increment_month_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False):
parent_tag = 'org' if org else 'user'
date = datetime.strptime(date_string, "%Y%m%d")
redis_name = "{0}:{1}:{2}:used_quota_month".format(parent_tag, entity_name, service)
redis_key_month = "{0}_{1}".format(date.year, date.month)
self.fake_redis_connection.hset(redis_name, redis_key_month, amount)
def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date=datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return user_service.UserService(geocoder_config, self.redis_conn)