Refactor python library to unify and rename as cdb_services

This commit is contained in:
Mario de Frutos
2016-01-29 11:54:50 +01:00
parent e7c58b9a51
commit 3960c13484
35 changed files with 397 additions and 872 deletions

View File

@@ -0,0 +1 @@
from geocoder import GoogleMapsGeocoder

View File

@@ -6,7 +6,7 @@ import googlemaps
from exceptions import MalformedResult
class Geocoder:
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
@@ -15,24 +15,25 @@ class Geocoder:
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
def geocode_address(self, searchtext, city=None, state=None,
country=None):
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext, components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
def _extract_lng_lat_from_result(self, result):
def geocode(self, searchtext, city=None, state=None,
country=None):
try:
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext,
components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
except KeyError:
raise MalformedResult()
def _extract_lng_lat_from_result(self, result):
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
def _build_optional_parameters(self, city=None, state=None,
country=None):
optional_params = {}

View File

@@ -0,0 +1 @@
from geocoder import HereMapsGeocoder

View File

@@ -0,0 +1,21 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class BadGeocodingParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Bad geocoding params: ' + json.dumps(self.value))
class NoGeocodingParams(Exception):
def __str__(self):
return repr('No params for geocoding specified')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

View File

@@ -4,13 +4,10 @@
import json
import requests
from heremaps.exception import BadGeocodingParams
from heremaps.exception import EmptyGeocoderResponse
from heremaps.exception import NoGeocodingParams
from heremaps.exception import MalformedResult
from exceptions import *
class Geocoder:
class HereMapsGeocoder:
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@@ -65,17 +62,19 @@ class Geocoder:
params[key] = value
if not params:
raise NoGeocodingParams()
return self.geocode(params)
return self._execute_geocode(params)
def _execute_geocode(self, params):
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
raise BadGeocodingParams(params)
try:
response = self._perform_request(params)
results = response['Response']['View'][0]['Result']
results = response['Response']['View'][0]['Result'][0]
return self._extract_lng_lat_from_result(results)
except IndexError:
return []
except KeyError:
raise MalformedResult()
def _perform_request(self, params):
request_params = {
@@ -92,11 +91,7 @@ class Geocoder:
response.raise_for_status()
def _extract_lng_lat_from_result(self, result):
try:
location = result['Location']
except KeyError:
raise MalformedResult()
location = result['Location']
longitude = location['DisplayPosition']['Longitude']
latitude = location['DisplayPosition']['Latitude']

View File

@@ -0,0 +1,3 @@
from config import GeocoderConfig, ConfigException
from quota import QuotaService
from user import UserMetricsService

View File

@@ -1,4 +1,4 @@
import user_service
from user import UserMetricsService
from datetime import date
@@ -8,7 +8,7 @@ class QuotaService:
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._user_service = user_service.UserService(
self._user_service = UserMetricsService(
self._user_geocoder_config, redis_connection)
def check_user_quota(self):

View File

@@ -2,7 +2,7 @@ from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
class UserService:
class UserMetricsService:
""" Class to manage all the user info """
SERVICE_GEOCODER_NOKIA = 'geocoder_here'

View File

@@ -0,0 +1 @@
from redis_tools import RedisConnection

View File

@@ -1,7 +1,7 @@
from redis.sentinel import Sentinel
class RedisHelper:
class RedisConnection:
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds

View File

@@ -1,11 +1,13 @@
redis==2.10.5
hiredis==0.1.5
# Dependency with incsv in the import
python-dateutil==2.2
heremaps==0.0.1
googlegeocoder==0.0.1
googlemaps
googlemaps==2.4.2
# Dependency for googlemaps package
requests<=2.9.1
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7
requests-mock==0.7.0

View File

@@ -1,5 +1,5 @@
"""
CartoDB Geocoder Python Library
CartoDB Services Python Library
See:
https://github.com/CartoDB/geocoder-api
@@ -8,11 +8,11 @@ https://github.com/CartoDB/geocoder-api
from setuptools import setup, find_packages
setup(
name='cartodb_geocoder',
name='cartodb_services',
version='0.0.1',
version='0.1.0',
description='CartoDB Geocoder Python Library',
description='CartoDB Services API Python Library',
url='https://github.com/CartoDB/geocoder-api',
@@ -29,7 +29,7 @@ setup(
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools geocoder',
keywords='maps api mapping tools geocoder routing',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),

View File

@@ -1,41 +1,41 @@
import test_helper
from cartodb_geocoder import config_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfigHelper(TestCase):
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(config_helper.ConfigException,
config_helper.GeocoderConfig,
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

View File

@@ -0,0 +1,119 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
from cartodb_services.google.exceptions import NoGeocodingParams
from cartodb_services.google.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json'
EMPTY_RESPONSE = """{
"results" : [],
"status" : "ZERO_RESULTS"
}"""
GOOD_RESPONSE = """{
"results" : [
{
"address_components" : [
{
"long_name" : "1600",
"short_name" : "1600",
"types" : [ "street_number" ]
},
{
"long_name" : "Amphitheatre Pkwy",
"short_name" : "Amphitheatre Pkwy",
"types" : [ "route" ]
},
{
"long_name" : "Mountain View",
"short_name" : "Mountain View",
"types" : [ "locality", "political" ]
},
{
"long_name" : "Santa Clara County",
"short_name" : "Santa Clara County",
"types" : [ "administrative_area_level_2", "political" ]
},
{
"long_name" : "California",
"short_name" : "CA",
"types" : [ "administrative_area_level_1", "political" ]
},
{
"long_name" : "United States",
"short_name" : "US",
"types" : [ "country", "political" ]
},
{
"long_name" : "94043",
"short_name" : "94043",
"types" : [ "postal_code" ]
}
],
"formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
},
"location_type" : "ROOFTOP",
"viewport" : {
"northeast" : {
"lat" : 37.4238253802915,
"lng" : -122.0829009197085
},
"southwest" : {
"lat" : 37.4211274197085,
"lng" : -122.0855988802915
}
}
},
"place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types" : [ "street_address" ]
}
],
"status" : "OK"
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -122.0842499)
self.assertEqual(response[1], 37.4224764)
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -2,25 +2,28 @@
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from heremaps import heremapsgeocoder
from heremaps.heremapsexceptions import BadGeocodingParams
from heremaps.heremapsexceptions import EmptyGeocoderResponse
from heremaps.heremapsexceptions import NoGeocodingParams
from heremaps.heremapsexceptions import MalformedResult
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
from cartodb_services.here.exceptions import NoGeocodingParams
from cartodb_services.here.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
class GeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = {
@requests_mock.Mocker()
class HereMapsGeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:31:57.273+0000"
},
"View": []
}
}
}"""
GOOD_RESPONSE = {
GOOD_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:30:32.187+0000"
@@ -28,7 +31,7 @@ class GeocoderTestCase(unittest.TestCase):
"View": [{
"_type": "SearchResultsViewType",
"ViewId": 0,
"Result": [{
"Result": {
"Relevance": 0.89,
"MatchLevel": "street",
"MatchQuality": {
@@ -57,7 +60,7 @@ class GeocoderTestCase(unittest.TestCase):
}
},
"Address": {
"Label": "Calle de Eloy Gonzalo, Madrid, España",
"Label": "Calle de Eloy Gonzalo, Madrid, Espana",
"Country": "ESP",
"State": "Comunidad de Madrid",
"County": "Madrid",
@@ -65,7 +68,7 @@ class GeocoderTestCase(unittest.TestCase):
"District": "Trafalgar",
"Street": "Calle de Eloy Gonzalo",
"AdditionalData": [{
"value": "España",
"value": "Espana",
"key": "CountryName"
},
{
@@ -78,45 +81,52 @@ class GeocoderTestCase(unittest.TestCase):
}]
}
}
}]
}
}]
}
}
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = heremapsgeocoder.Geocoder(None, None)
self.geocoder = HereMapsGeocoder(None, None)
def test_geocode_address_with_valid_params(self):
self.geocoder.perform_request = lambda x: self.GOOD_RESPONSE
response = self.geocoder.geocode_address(
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
def test_geocode_address_with_invalid_params(self):
self.assertEqual(response[0], -3.70126)
self.assertEqual(response[1], 40.43433)
def test_geocode_address_with_invalid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(BadGeocodingParams):
self.geocoder.geocode_address(
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
manolo='escobar')
def test_geocode_address_with_no_params(self):
def test_geocode_address_with_no_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(NoGeocodingParams):
self.geocoder.geocode_address()
self.geocoder.geocode()
def test_geocode_address_empty_response(self):
self.geocoder.perform_request = lambda x: self.EMPTY_RESPONSE
with self.assertRaises(EmptyGeocoderResponse):
self.geocoder.geocode_address(searchtext='lkajfñlasjfñ')
def test_extract_lng_lat_from_result(self):
result = self.GOOD_RESPONSE['Response']['View'][0]['Result'][0]
coordinates = self.geocoder.extract_lng_lat_from_result(result)
self.assertEqual(coordinates[0], -3.70126)
self.assertEqual(coordinates[1], 40.43433)
def test_extract_lng_lat_from_result_with_malformed_result(self):
result = {'manolo': 'escobar'}
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.extract_lng_lat_from_result(result)
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import quota_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -14,10 +14,6 @@ class TestQuotaService(TestCase):
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
# def increment_geocoder_uses(self, username, orgname=None,
# date=date.today(), service='geocoder_here',
# metric='success_responses', amount=20):
def setUp(self):
self.redis_conn = MockRedis()
@@ -68,28 +64,19 @@ class TestQuotaService(TestCase):
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
total_requests = self.redis_conn.zscore(name, date.today().day)
assert total_requests == 3
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month)
org_total_requests = self.redis_conn.zscore(org_name, date.today().day)
assert org_total_requests == 3
user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
user_total_requests = self.redis_conn.zscore(user_name, date.today().day)
assert user_total_requests == 3
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
@@ -101,9 +88,9 @@ class TestQuotaService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return quota_service.QuotaService(geocoder_config,
redis_connection = self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import user_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
@@ -84,7 +84,7 @@ class TestUserService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return user_service.UserService(geocoder_config, self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return UserMetricsService(geocoder_config, self.redis_conn)

View File

@@ -1,7 +0,0 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
class MalformedResult(Exception):
def __str__(self):
return repr('Malformed result. The API might have changed.')

View File

@@ -1,41 +0,0 @@
"""
A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension
See:
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='google-geocoder',
version='0.0.1',
description='A Google Maps wrapper for the CartoDB geocoder server PostgreSQL extension',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 2 - Beta',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest'],
}
)

View File

@@ -1,4 +0,0 @@
requests==2.9.1
# Test
nose==1.3.7

View File

@@ -1,41 +0,0 @@
"""
A Here Maps API Python wrapper
See:
https://developer.here.com
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='heremaps',
version='0.0.1',
description='A Here Maps API Python wrapper',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 5 - Production',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest'],
}
)