Refactor python library to unify and rename as cdb_services

This commit is contained in:
Mario de Frutos
2016-01-29 11:54:50 +01:00
parent e7c58b9a51
commit 3960c13484
35 changed files with 397 additions and 872 deletions

View File

@@ -0,0 +1,41 @@
import test_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit is False
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

View File

@@ -0,0 +1,119 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
from cartodb_services.google.exceptions import NoGeocodingParams
from cartodb_services.google.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json'
EMPTY_RESPONSE = """{
"results" : [],
"status" : "ZERO_RESULTS"
}"""
GOOD_RESPONSE = """{
"results" : [
{
"address_components" : [
{
"long_name" : "1600",
"short_name" : "1600",
"types" : [ "street_number" ]
},
{
"long_name" : "Amphitheatre Pkwy",
"short_name" : "Amphitheatre Pkwy",
"types" : [ "route" ]
},
{
"long_name" : "Mountain View",
"short_name" : "Mountain View",
"types" : [ "locality", "political" ]
},
{
"long_name" : "Santa Clara County",
"short_name" : "Santa Clara County",
"types" : [ "administrative_area_level_2", "political" ]
},
{
"long_name" : "California",
"short_name" : "CA",
"types" : [ "administrative_area_level_1", "political" ]
},
{
"long_name" : "United States",
"short_name" : "US",
"types" : [ "country", "political" ]
},
{
"long_name" : "94043",
"short_name" : "94043",
"types" : [ "postal_code" ]
}
],
"formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
},
"location_type" : "ROOFTOP",
"viewport" : {
"northeast" : {
"lat" : 37.4238253802915,
"lng" : -122.0829009197085
},
"southwest" : {
"lat" : 37.4211274197085,
"lng" : -122.0855988802915
}
}
},
"place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types" : [ "street_address" ]
}
],
"status" : "OK"
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -122.0842499)
self.assertEqual(response[1], 37.4224764)
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -0,0 +1,33 @@
from datetime import datetime, date
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
service="heremaps",
end_date=datetime.today()):
user_redis_name = "rails:users:{0}".format(username)
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
def build_redis_org_config(redis_conn, orgname, quota=100,
end_date=datetime.today()):
org_redis_name = "rails:orgs:{0}".format(orgname)
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
def increment_geocoder_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
prefix = 'org' if orgname else 'user'
entity_name = orgname if orgname else username
yearmonth = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service, metric, yearmonth)
redis_conn.zincrby(redis_name, date.day, amount)

View File

@@ -0,0 +1,132 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
from cartodb_services.here.exceptions import NoGeocodingParams
from cartodb_services.here.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class HereMapsGeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:31:57.273+0000"
},
"View": []
}
}"""
GOOD_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:30:32.187+0000"
},
"View": [{
"_type": "SearchResultsViewType",
"ViewId": 0,
"Result": {
"Relevance": 0.89,
"MatchLevel": "street",
"MatchQuality": {
"City": 1.0,
"Street": [1.0]
},
"Location": {
"LocationId": "NT_yyKB4r3mCWAX4voWgxPcuA",
"LocationType": "address",
"DisplayPosition": {
"Latitude": 40.43433,
"Longitude": -3.70126
},
"NavigationPosition": [{
"Latitude": 40.43433,
"Longitude": -3.70126
}],
"MapView": {
"TopLeft": {
"Latitude": 40.43493,
"Longitude": -3.70404
},
"BottomRight": {
"Latitude": 40.43373,
"Longitude": -3.69873
}
},
"Address": {
"Label": "Calle de Eloy Gonzalo, Madrid, Espana",
"Country": "ESP",
"State": "Comunidad de Madrid",
"County": "Madrid",
"City": "Madrid",
"District": "Trafalgar",
"Street": "Calle de Eloy Gonzalo",
"AdditionalData": [{
"value": "Espana",
"key": "CountryName"
},
{
"value": "Comunidad de Madrid",
"key": "StateName"
},
{
"value": "Madrid",
"key": "CountyName"
}]
}
}
}
}]
}
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = HereMapsGeocoder(None, None)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -3.70126)
self.assertEqual(response[1], 40.43433)
def test_geocode_address_with_invalid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(BadGeocodingParams):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
manolo='escobar')
def test_geocode_address_with_no_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(NoGeocodingParams):
self.geocoder.geocode()
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -0,0 +1,96 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
class TestQuotaService(TestCase):
# single user
# user:<username>:<service>:<metric>:YYYYMM:DD
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service('test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date = datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota = quota, service = service,
soft_limit = soft_limit,
end_date = end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

View File

@@ -0,0 +1,90 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
from datetime import timedelta
class TestUserService(TestCase):
NOKIA_GEOCODER = 'geocoder_here'
def setUp(self):
self.redis_conn = MockRedis()
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service('test_user')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_org_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service('test_user', orgname='test_org')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_should_increment_user_used_quota_for_one_date(self):
us = self.__build_user_service('test_user')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_should_increment_org_used_quota(self):
us = self.__build_user_service('test_user', orgname='test_org')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_should_increment_user_used_quota_in_for_multiples_dates(self):
two_days_ago = datetime.today() - timedelta(days=2)
one_day_ago = datetime.today() - timedelta(days=1)
one_day_after = datetime.today() + timedelta(days=1)
us = self.__build_user_service('test_user', end_date=one_day_ago)
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
date=date.today())
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=two_days_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_after)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date=datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return UserMetricsService(geocoder_config, self.redis_conn)