cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor python library to unify and rename as cdb_services

Browse Source
This commit is contained in:
Mario de Frutos
2016-01-29 11:54:50 +01:00
parent e7c58b9a51
commit 3960c13484
35 changed files with 397 additions and 872 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
server/lib/python/cartodb_services/cartodb_services/__init__.py Normal file
View File

1
server/lib/python/cartodb_services/cartodb_services/google/__init__.py Normal file
View File

@@ -0,0 +1 @@
from geocoder import GoogleMapsGeocoder

21
server/lib/python/cartodb_services/cartodb_services/google/exceptions.py Normal file
View File

@@ -0,0 +1,21 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class BadGeocodingParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Bad geocoding params: ' + json.dumps(self.value))
class NoGeocodingParams(Exception):
def __str__(self):
return repr('No params for geocoding specified')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

50
server/lib/python/cartodb_services/cartodb_services/google/geocoder.py Normal file
View File

@@ -0,0 +1,50 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import googlemaps
from exceptions import MalformedResult
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
self.client_id = self._clean_client_id(client_id)
self.client_secret = client_secret
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
def geocode(self, searchtext, city=None, state=None,
country=None):
try:
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext,
components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
except KeyError:
raise MalformedResult()
def _extract_lng_lat_from_result(self, result):
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
def _build_optional_parameters(self, city=None, state=None,
country=None):
optional_params = {}
if city:
optional_params['locality'] = city
if state:
optional_params['administrative_area'] = state
if country:
optional_params['country'] = country
return optional_params
def _clean_client_id(self, client_id):
# Consistency with how the client_id is saved in metadata
return client_id.replace('client=', '')

1
server/lib/python/cartodb_services/cartodb_services/here/__init__.py Normal file
View File

@@ -0,0 +1 @@
from geocoder import HereMapsGeocoder

21
server/lib/python/cartodb_services/cartodb_services/here/exceptions.py Normal file
View File

@@ -0,0 +1,21 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class BadGeocodingParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Bad geocoding params: ' + json.dumps(self.value))
class NoGeocodingParams(Exception):
def __str__(self):
return repr('No params for geocoding specified')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

98
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py Normal file
View File

@@ -0,0 +1,98 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
import requests
from exceptions import *
class HereMapsGeocoder:
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
DEFAULT_MAXRESULTS = 1
DEFAULT_GEN = 9
ADDRESS_PARAMS = [
'city',
'country',
'county',
'district',
'housenumber',
'postalcode',
'searchtext',
'state',
'street'
]
ADMITTED_PARAMS = [
'additionaldata',
'app_id',
'app_code',
'bbox',
'countryfocus',
'gen',
'jsonattributes',
'jsoncallback',
'language',
'locationattributes',
'locationid',
'mapview',
'maxresults',
'pageinformation',
'politicalview',
'prox',
'strictlanguagemode'
] + ADDRESS_PARAMS
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
self.app_id = app_id
self.app_code = app_code
self.maxresults = maxresults
self.gen = gen
self.host = host
def geocode(self, **kwargs):
params = {}
for key, value in kwargs.iteritems():
if value:
params[key] = value
if not params:
raise NoGeocodingParams()
return self._execute_geocode(params)
def _execute_geocode(self, params):
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
raise BadGeocodingParams(params)
try:
response = self._perform_request(params)
results = response['Response']['View'][0]['Result'][0]
return self._extract_lng_lat_from_result(results)
except IndexError:
return []
except KeyError:
raise MalformedResult()
def _perform_request(self, params):
request_params = {
'app_id': self.app_id,
'app_code': self.app_code,
'maxresults': self.maxresults,
'gen': self.gen
}
request_params.update(params)
response = requests.get(self.host, params=request_params)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
else:
response.raise_for_status()
def _extract_lng_lat_from_result(self, result):
location = result['Location']
longitude = location['DisplayPosition']['Longitude']
latitude = location['DisplayPosition']['Latitude']
return [longitude, latitude]

3
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from config import GeocoderConfig, ConfigException
from quota import QuotaService
from user import UserMetricsService

148
server/lib/python/cartodb_services/cartodb_services/metrics/config.py Normal file
View File

@@ -0,0 +1,148 @@
import json
from dateutil.parser import parse as date_parse
class ConfigException(Exception):
pass
class GeocoderConfig:
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date',
'heremaps_app_id', 'heremaps_app_code', 'username',
'orgname']
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit',
'heremaps_app_id', 'heremaps_app_code']
NOKIA_GEOCODER = 'heremaps'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
GOOGLE_GEOCODER = 'google'
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
GEOCODER_TYPE = 'geocoder_type'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, username, orgname=None,
heremaps_app_id=None, heremaps_app_code=None):
self._redis_connection = redis_connection
config = self.__get_user_config(username, orgname, heremaps_app_id,
heremaps_app_code)
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
heremaps_app_code=None):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
user_config[self.USERNAME_KEY] = username
user_config[self.ORGNAME_KEY] = orgname
user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id
user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code
if orgname:
self.__get_organization_config(orgname, user_config)
return user_config
def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
"rails:orgs:{0}".format(orgname))
if not org_config:
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]:
raise ConfigException("""Nokia geocoder configuration is missing. Check it please""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
return True
def __parse_config(self, filtered_config):
self._username = filtered_config[self.USERNAME_KEY].lower()
if filtered_config[self.ORGNAME_KEY]:
self._orgname = filtered_config[self.ORGNAME_KEY].lower()
else:
self._orgname = None
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_geocoding_limit = True
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY]
self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
@property
def service_type(self):
if self._geocoder_type == self.GOOGLE_GEOCODER:
return 'geocoder_google'
else:
return 'geocoder_here'
@property
def heremaps_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@property
def google_api_key(self):
return self._google_maps_api_key
@property
def geocoding_quota(self):
return self._geocoding_quota
@property
def soft_geocoding_limit(self):
return self._soft_geocoding_limit
@property
def period_end_date(self):
return self._period_end_date
@property
def heremaps_app_id(self):
return self._heremaps_app_id
@property
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def username(self):
return self._username
@property
def organization(self):
return self._orgname

49
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py Normal file
View File

@@ -0,0 +1,49 @@
from user import UserMetricsService
from datetime import date
class QuotaService:
""" Class to manage all the quota operation for
the Geocoder SQL API Extension """
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._user_service = UserMetricsService(
self._user_geocoder_config, redis_connection)
def check_user_quota(self):
""" Check if the current user quota surpasses the current quota """
# We don't have quota check for google geocoder
if self._user_geocoder_config.google_geocoder:
return True
user_quota = self._user_geocoder_config.geocoding_quota
today = date.today()
service_type = self._user_geocoder_config.service_type
current_used = self._user_service.used_quota(service_type, today)
soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit
if soft_geocoding_limit or current_used <= user_quota:
return True
else:
return False
def increment_success_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "success_responses",
amount=amount)
def increment_empty_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "empty_responses",
amount=amount)
def increment_failed_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "fail_responses",
amount=amount)
def increment_total_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "total_requests",
amount=amount)

96
server/lib/python/cartodb_services/cartodb_services/metrics/user.py Normal file
View File

@@ -0,0 +1,96 @@
from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
class UserMetricsService:
""" Class to manage all the user info """
SERVICE_GEOCODER_NOKIA = 'geocoder_here'
SERVICE_GEOCODER_GOOGLE = 'geocoder_google'
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
GEOCODING_QUOTA_KEY = "geocoding_quota"
GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit"
REDIS_CONNECTION_KEY = "redis_connection"
REDIS_CONNECTION_HOST = "redis_host"
REDIS_CONNECTION_PORT = "redis_port"
REDIS_CONNECTION_DB = "redis_db"
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._redis_connection = redis_connection
self._username = user_geocoder_config.username
self._orgname = user_geocoder_config.organization
def used_quota(self, service_type, date):
""" Recover the used quota for the user in the current month """
date_from, date_to = self.__current_billing_cycle()
current_use = 0
success_responses = self.get_metrics(service_type,
'success_responses', date_from,
date_to)
empty_responses = self.get_metrics(service_type,
'empty_responses', date_from,
date_to)
current_use += (success_responses + empty_responses)
if service_type == self.SERVICE_GEOCODER_NOKIA:
cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE,
'success_responses', date_from,
date_to)
current_use += cache_hits
return current_use
def increment_service_use(self, service_type, metric, date=date.today(), amount=1):
""" Increment the services uses in monthly and daily basis"""
self.__increment_user_uses(service_type, metric, date, amount)
if self._orgname:
self.__increment_organization_uses(service_type, metric, date, amount)
def get_metrics(self, service, metric, date_from, date_to):
aggregated_metric = 0
key_prefix = "org" if self._orgname else "user"
entity_name = self._orgname if self._orgname else self._username
for date in self.__generate_date_range(date_from, date_to):
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
service, metric, date)
score = self._redis_connection.zscore(redis_prefix, date.day)
aggregated_metric += score if score else 0
return aggregated_metric
# Private functions
def __increment_user_uses(self, service_type, metric, date, amount):
redis_prefix = self.__parse_redis_prefix("user", self._username,
service_type, metric, date)
self._redis_connection.zincrby(redis_prefix, date.day, amount)
def __increment_organization_uses(self, service_type, metric, date, amount):
redis_prefix = self.__parse_redis_prefix("org", self._orgname,
service_type, metric, date)
self._redis_connection.zincrby(redis_prefix, date.day, amount)
def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date):
yearmonth_key = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service_type, metric,
yearmonth_key)
return redis_name
def __current_billing_cycle(self):
""" Return the begining and end date for the current billing cycle """
end_period_day = self._user_geocoder_config.period_end_date.day
today = date.today()
if end_period_day > today.day:
temp_date = today + relativedelta(months=-1)
date_from = date(temp_date.year, temp_date.month, end_period_day)
else:
date_from = date(today.year, today.month, end_period_day)
return date_from, today
def __generate_date_range(self, date_from, date_to):
for n in range(int((date_to - date_from).days + 1)):
yield date_from + timedelta(n)

1
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py Normal file
View File

@@ -0,0 +1 @@
from redis_tools import RedisConnection

29
server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py Normal file
View File

@@ -0,0 +1,29 @@
from redis.sentinel import Sentinel
class RedisConnection:
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds
REDIS_SENTINEL_DEFAULT_PORT = 26379
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.sentinel_host = sentinel_host
self.sentinel_port = sentinel_port
self.sentinel_master_id = sentinel_master_id
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def redis_connection(self):
return self.__create_redis_connection()
def __create_redis_connection(self):
sentinel = Sentinel([(self.sentinel_host,
self.REDIS_SENTINEL_DEFAULT_PORT)],
socket_timeout=self.timeout)
return sentinel.master_for(
self.sentinel_master_id,
socket_timeout=self.timeout,
db=self.redis_db
)

13
server/lib/python/cartodb_services/requirements.txt Normal file
View File

@@ -0,0 +1,13 @@
redis==2.10.5
hiredis==0.1.5
# Dependency with incsv in the import
python-dateutil==2.2
googlemaps==2.4.2
# Dependency for googlemaps package
requests<=2.9.1
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7
requests-mock==0.7.0

40
server/lib/python/cartodb_services/setup.py Normal file
View File

@@ -0,0 +1,40 @@
"""
CartoDB Services Python Library
See:
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.1.0',
description='CartoDB Services API Python Library',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 4 - Beta',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools geocoder routing',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest', 'nose', 'mockredispy', 'mock'],
}
)

41
server/lib/python/cartodb_services/test/test_config.py Normal file
View File

@@ -0,0 +1,41 @@
import test_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit is False
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

119
server/lib/python/cartodb_services/test/test_googlegeocoder.py Normal file
View File

@@ -0,0 +1,119 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
from cartodb_services.google.exceptions import NoGeocodingParams
from cartodb_services.google.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json'
EMPTY_RESPONSE = """{
"results" : [],
"status" : "ZERO_RESULTS"
}"""
GOOD_RESPONSE = """{
"results" : [
{
"address_components" : [
{
"long_name" : "1600",
"short_name" : "1600",
"types" : [ "street_number" ]
},
{
"long_name" : "Amphitheatre Pkwy",
"short_name" : "Amphitheatre Pkwy",
"types" : [ "route" ]
},
{
"long_name" : "Mountain View",
"short_name" : "Mountain View",
"types" : [ "locality", "political" ]
},
{
"long_name" : "Santa Clara County",
"short_name" : "Santa Clara County",
"types" : [ "administrative_area_level_2", "political" ]
},
{
"long_name" : "California",
"short_name" : "CA",
"types" : [ "administrative_area_level_1", "political" ]
},
{
"long_name" : "United States",
"short_name" : "US",
"types" : [ "country", "political" ]
},
{
"long_name" : "94043",
"short_name" : "94043",
"types" : [ "postal_code" ]
}
],
"formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
},
"location_type" : "ROOFTOP",
"viewport" : {
"northeast" : {
"lat" : 37.4238253802915,
"lng" : -122.0829009197085
},
"southwest" : {
"lat" : 37.4211274197085,
"lng" : -122.0855988802915
}
}
},
"place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types" : [ "street_address" ]
}
],
"status" : "OK"
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -122.0842499)
self.assertEqual(response[1], 37.4224764)
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

33
server/lib/python/cartodb_services/test/test_helper.py Normal file
View File

@@ -0,0 +1,33 @@
from datetime import datetime, date
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
service="heremaps",
end_date=datetime.today()):
user_redis_name = "rails:users:{0}".format(username)
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
def build_redis_org_config(redis_conn, orgname, quota=100,
end_date=datetime.today()):
org_redis_name = "rails:orgs:{0}".format(orgname)
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
def increment_geocoder_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
prefix = 'org' if orgname else 'user'
entity_name = orgname if orgname else username
yearmonth = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service, metric, yearmonth)
redis_conn.zincrby(redis_name, date.day, amount)

132
server/lib/python/cartodb_services/test/test_heremapsgeocoder.py Normal file
View File

@@ -0,0 +1,132 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
from cartodb_services.here.exceptions import NoGeocodingParams
from cartodb_services.here.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class HereMapsGeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:31:57.273+0000"
},
"View": []
}
}"""
GOOD_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:30:32.187+0000"
},
"View": [{
"_type": "SearchResultsViewType",
"ViewId": 0,
"Result": {
"Relevance": 0.89,
"MatchLevel": "street",
"MatchQuality": {
"City": 1.0,
"Street": [1.0]
},
"Location": {
"LocationId": "NT_yyKB4r3mCWAX4voWgxPcuA",
"LocationType": "address",
"DisplayPosition": {
"Latitude": 40.43433,
"Longitude": -3.70126
},
"NavigationPosition": [{
"Latitude": 40.43433,
"Longitude": -3.70126
}],
"MapView": {
"TopLeft": {
"Latitude": 40.43493,
"Longitude": -3.70404
},
"BottomRight": {
"Latitude": 40.43373,
"Longitude": -3.69873
}
},
"Address": {
"Label": "Calle de Eloy Gonzalo, Madrid, Espana",
"Country": "ESP",
"State": "Comunidad de Madrid",
"County": "Madrid",
"City": "Madrid",
"District": "Trafalgar",
"Street": "Calle de Eloy Gonzalo",
"AdditionalData": [{
"value": "Espana",
"key": "CountryName"
},
{
"value": "Comunidad de Madrid",
"key": "StateName"
},
{
"value": "Madrid",
"key": "CountyName"
}]
}
}
}
}]
}
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = HereMapsGeocoder(None, None)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -3.70126)
self.assertEqual(response[1], 40.43433)
def test_geocode_address_with_invalid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(BadGeocodingParams):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
manolo='escobar')
def test_geocode_address_with_no_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(NoGeocodingParams):
self.geocoder.geocode()
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

96
server/lib/python/cartodb_services/test/test_quota_service.py Normal file
View File

@@ -0,0 +1,96 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
class TestQuotaService(TestCase):
# single user
# user:<username>:<service>:<metric>:YYYYMM:DD
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service('test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date = datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota = quota, service = service,
soft_limit = soft_limit,
end_date = end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

90
server/lib/python/cartodb_services/test/test_user_service.py Normal file
View File

@@ -0,0 +1,90 @@
import test_helper
from mockredis import MockRedis
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
from datetime import timedelta
class TestUserService(TestCase):
NOKIA_GEOCODER = 'geocoder_here'
def setUp(self):
self.redis_conn = MockRedis()
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service('test_user')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_org_not_amount_in_used_quota_for_month_should_be_0(self):
us = self.__build_user_service('test_user', orgname='test_org')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
def test_should_increment_user_used_quota_for_one_date(self):
us = self.__build_user_service('test_user')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_should_increment_org_used_quota(self):
us = self.__build_user_service('test_user', orgname='test_org')
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def test_should_increment_user_used_quota_in_for_multiples_dates(self):
two_days_ago = datetime.today() - timedelta(days=2)
one_day_ago = datetime.today() - timedelta(days=1)
one_day_after = datetime.today() + timedelta(days=1)
us = self.__build_user_service('test_user', end_date=one_day_ago)
us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
date=date.today())
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=two_days_ago)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses',
date=one_day_after)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date=datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return UserMetricsService(geocoder_config, self.redis_conn)