Refactor python library to unify and rename as cdb_services
This commit is contained in:
Mario de Frutos
@@ -0,0 +1 @@
|
||||
from geocoder import GoogleMapsGeocoder
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
|
||||
|
||||
class BadGeocodingParams(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr('Bad geocoding params: ' + json.dumps(self.value))
|
||||
|
||||
|
||||
class NoGeocodingParams(Exception):
|
||||
def __str__(self):
|
||||
return repr('No params for geocoding specified')
|
||||
|
||||
|
||||
class MalformedResult(Exception):
|
||||
def __str__(self):
|
||||
return repr('Result structure is malformed')
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import googlemaps
|
||||
|
||||
from exceptions import MalformedResult
|
||||
|
||||
|
||||
class GoogleMapsGeocoder:
|
||||
"""A Google Maps Geocoder wrapper for python"""
|
||||
|
||||
def __init__(self, client_id, client_secret):
|
||||
self.client_id = self._clean_client_id(client_id)
|
||||
self.client_secret = client_secret
|
||||
self.geocoder = googlemaps.Client(
|
||||
client_id=self.client_id, client_secret=self.client_secret)
|
||||
|
||||
def geocode(self, searchtext, city=None, state=None,
|
||||
country=None):
|
||||
try:
|
||||
opt_params = self._build_optional_parameters(city, state, country)
|
||||
results = self.geocoder.geocode(address=searchtext,
|
||||
components=opt_params)
|
||||
if results:
|
||||
return self._extract_lng_lat_from_result(results[0])
|
||||
else:
|
||||
return []
|
||||
except KeyError:
|
||||
raise MalformedResult()
|
||||
|
||||
def _extract_lng_lat_from_result(self, result):
|
||||
location = result['geometry']['location']
|
||||
longitude = location['lng']
|
||||
latitude = location['lat']
|
||||
return [longitude, latitude]
|
||||
|
||||
def _build_optional_parameters(self, city=None, state=None,
|
||||
country=None):
|
||||
optional_params = {}
|
||||
if city:
|
||||
optional_params['locality'] = city
|
||||
if state:
|
||||
optional_params['administrative_area'] = state
|
||||
if country:
|
||||
optional_params['country'] = country
|
||||
return optional_params
|
||||
|
||||
def _clean_client_id(self, client_id):
|
||||
# Consistency with how the client_id is saved in metadata
|
||||
return client_id.replace('client=', '')
|
||||
@@ -0,0 +1 @@
|
||||
from geocoder import HereMapsGeocoder
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
|
||||
|
||||
class BadGeocodingParams(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr('Bad geocoding params: ' + json.dumps(self.value))
|
||||
|
||||
|
||||
class NoGeocodingParams(Exception):
|
||||
def __str__(self):
|
||||
return repr('No params for geocoding specified')
|
||||
|
||||
|
||||
class MalformedResult(Exception):
|
||||
def __str__(self):
|
||||
return repr('Result structure is malformed')
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import requests
|
||||
|
||||
from exceptions import *
|
||||
|
||||
|
||||
class HereMapsGeocoder:
|
||||
'A Here Maps Geocoder wrapper for python'
|
||||
|
||||
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
|
||||
STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json'
|
||||
DEFAULT_MAXRESULTS = 1
|
||||
DEFAULT_GEN = 9
|
||||
|
||||
ADDRESS_PARAMS = [
|
||||
'city',
|
||||
'country',
|
||||
'county',
|
||||
'district',
|
||||
'housenumber',
|
||||
'postalcode',
|
||||
'searchtext',
|
||||
'state',
|
||||
'street'
|
||||
]
|
||||
|
||||
ADMITTED_PARAMS = [
|
||||
'additionaldata',
|
||||
'app_id',
|
||||
'app_code',
|
||||
'bbox',
|
||||
'countryfocus',
|
||||
'gen',
|
||||
'jsonattributes',
|
||||
'jsoncallback',
|
||||
'language',
|
||||
'locationattributes',
|
||||
'locationid',
|
||||
'mapview',
|
||||
'maxresults',
|
||||
'pageinformation',
|
||||
'politicalview',
|
||||
'prox',
|
||||
'strictlanguagemode'
|
||||
] + ADDRESS_PARAMS
|
||||
|
||||
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
|
||||
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
|
||||
self.app_id = app_id
|
||||
self.app_code = app_code
|
||||
self.maxresults = maxresults
|
||||
self.gen = gen
|
||||
self.host = host
|
||||
|
||||
def geocode(self, **kwargs):
|
||||
params = {}
|
||||
for key, value in kwargs.iteritems():
|
||||
if value:
|
||||
params[key] = value
|
||||
if not params:
|
||||
raise NoGeocodingParams()
|
||||
return self._execute_geocode(params)
|
||||
|
||||
def _execute_geocode(self, params):
|
||||
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
|
||||
raise BadGeocodingParams(params)
|
||||
try:
|
||||
response = self._perform_request(params)
|
||||
results = response['Response']['View'][0]['Result'][0]
|
||||
return self._extract_lng_lat_from_result(results)
|
||||
except IndexError:
|
||||
return []
|
||||
except KeyError:
|
||||
raise MalformedResult()
|
||||
|
||||
def _perform_request(self, params):
|
||||
request_params = {
|
||||
'app_id': self.app_id,
|
||||
'app_code': self.app_code,
|
||||
'maxresults': self.maxresults,
|
||||
'gen': self.gen
|
||||
}
|
||||
request_params.update(params)
|
||||
response = requests.get(self.host, params=request_params)
|
||||
if response.status_code == requests.codes.ok:
|
||||
return json.loads(response.text)
|
||||
else:
|
||||
response.raise_for_status()
|
||||
|
||||
def _extract_lng_lat_from_result(self, result):
|
||||
location = result['Location']
|
||||
longitude = location['DisplayPosition']['Longitude']
|
||||
latitude = location['DisplayPosition']['Latitude']
|
||||
|
||||
return [longitude, latitude]
|
||||
@@ -0,0 +1,3 @@
|
||||
from config import GeocoderConfig, ConfigException
|
||||
from quota import QuotaService
|
||||
from user import UserMetricsService
|
||||
@@ -0,0 +1,148 @@
|
||||
import json
|
||||
from dateutil.parser import parse as date_parse
|
||||
|
||||
|
||||
class ConfigException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class GeocoderConfig:
|
||||
|
||||
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
|
||||
'geocoding_quota', 'soft_geocoding_limit',
|
||||
'geocoder_type', 'period_end_date',
|
||||
'heremaps_app_id', 'heremaps_app_code', 'username',
|
||||
'orgname']
|
||||
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit',
|
||||
'heremaps_app_id', 'heremaps_app_code']
|
||||
NOKIA_GEOCODER = 'heremaps'
|
||||
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
|
||||
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
|
||||
GOOGLE_GEOCODER = 'google'
|
||||
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
|
||||
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
|
||||
GEOCODER_TYPE = 'geocoder_type'
|
||||
QUOTA_KEY = 'geocoding_quota'
|
||||
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
|
||||
USERNAME_KEY = 'username'
|
||||
ORGNAME_KEY = 'orgname'
|
||||
PERIOD_END_DATE = 'period_end_date'
|
||||
|
||||
def __init__(self, redis_connection, username, orgname=None,
|
||||
heremaps_app_id=None, heremaps_app_code=None):
|
||||
self._redis_connection = redis_connection
|
||||
config = self.__get_user_config(username, orgname, heremaps_app_id,
|
||||
heremaps_app_code)
|
||||
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
|
||||
self.__check_config(filtered_config)
|
||||
self.__parse_config(filtered_config)
|
||||
|
||||
def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
|
||||
heremaps_app_code=None):
|
||||
user_config = self._redis_connection.hgetall(
|
||||
"rails:users:{0}".format(username))
|
||||
if not user_config:
|
||||
raise ConfigException("""There is no user config available. Please check your configuration.'""")
|
||||
else:
|
||||
user_config[self.USERNAME_KEY] = username
|
||||
user_config[self.ORGNAME_KEY] = orgname
|
||||
user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id
|
||||
user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code
|
||||
if orgname:
|
||||
self.__get_organization_config(orgname, user_config)
|
||||
|
||||
return user_config
|
||||
|
||||
def __get_organization_config(self, orgname, user_config):
|
||||
org_config = self._redis_connection.hgetall(
|
||||
"rails:orgs:{0}".format(orgname))
|
||||
if not org_config:
|
||||
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
|
||||
else:
|
||||
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
|
||||
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
|
||||
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
||||
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
|
||||
def __check_config(self, filtered_config):
|
||||
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
||||
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
|
||||
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
|
||||
if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]:
|
||||
raise ConfigException("""Nokia geocoder configuration is missing. Check it please""")
|
||||
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
||||
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
||||
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
|
||||
|
||||
return True
|
||||
|
||||
def __parse_config(self, filtered_config):
|
||||
self._username = filtered_config[self.USERNAME_KEY].lower()
|
||||
if filtered_config[self.ORGNAME_KEY]:
|
||||
self._orgname = filtered_config[self.ORGNAME_KEY].lower()
|
||||
else:
|
||||
self._orgname = None
|
||||
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
|
||||
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
|
||||
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
||||
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
||||
self._soft_geocoding_limit = True
|
||||
else:
|
||||
self._soft_geocoding_limit = False
|
||||
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
||||
self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY]
|
||||
self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]
|
||||
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
||||
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
|
||||
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
if self._geocoder_type == self.GOOGLE_GEOCODER:
|
||||
return 'geocoder_google'
|
||||
else:
|
||||
return 'geocoder_here'
|
||||
|
||||
@property
|
||||
def heremaps_geocoder(self):
|
||||
return self._geocoder_type == self.NOKIA_GEOCODER
|
||||
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
return self._geocoder_type == self.GOOGLE_GEOCODER
|
||||
|
||||
@property
|
||||
def google_client_id(self):
|
||||
return self._google_maps_client_id
|
||||
|
||||
@property
|
||||
def google_api_key(self):
|
||||
return self._google_maps_api_key
|
||||
|
||||
@property
|
||||
def geocoding_quota(self):
|
||||
return self._geocoding_quota
|
||||
|
||||
@property
|
||||
def soft_geocoding_limit(self):
|
||||
return self._soft_geocoding_limit
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@property
|
||||
def heremaps_app_id(self):
|
||||
return self._heremaps_app_id
|
||||
|
||||
@property
|
||||
def heremaps_app_code(self):
|
||||
return self._heremaps_app_code
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._orgname
|
||||
@@ -0,0 +1,49 @@
|
||||
from user import UserMetricsService
|
||||
from datetime import date
|
||||
|
||||
|
||||
class QuotaService:
|
||||
""" Class to manage all the quota operation for
|
||||
the Geocoder SQL API Extension """
|
||||
|
||||
def __init__(self, user_geocoder_config, redis_connection):
|
||||
self._user_geocoder_config = user_geocoder_config
|
||||
self._user_service = UserMetricsService(
|
||||
self._user_geocoder_config, redis_connection)
|
||||
|
||||
def check_user_quota(self):
|
||||
""" Check if the current user quota surpasses the current quota """
|
||||
# We don't have quota check for google geocoder
|
||||
if self._user_geocoder_config.google_geocoder:
|
||||
return True
|
||||
|
||||
user_quota = self._user_geocoder_config.geocoding_quota
|
||||
today = date.today()
|
||||
service_type = self._user_geocoder_config.service_type
|
||||
current_used = self._user_service.used_quota(service_type, today)
|
||||
soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit
|
||||
|
||||
if soft_geocoding_limit or current_used <= user_quota:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def increment_success_geocoder_use(self, amount=1):
|
||||
self._user_service.increment_service_use(
|
||||
self._user_geocoder_config.service_type, "success_responses",
|
||||
amount=amount)
|
||||
|
||||
def increment_empty_geocoder_use(self, amount=1):
|
||||
self._user_service.increment_service_use(
|
||||
self._user_geocoder_config.service_type, "empty_responses",
|
||||
amount=amount)
|
||||
|
||||
def increment_failed_geocoder_use(self, amount=1):
|
||||
self._user_service.increment_service_use(
|
||||
self._user_geocoder_config.service_type, "fail_responses",
|
||||
amount=amount)
|
||||
|
||||
def increment_total_geocoder_use(self, amount=1):
|
||||
self._user_service.increment_service_use(
|
||||
self._user_geocoder_config.service_type, "total_requests",
|
||||
amount=amount)
|
||||
@@ -0,0 +1,96 @@
|
||||
from datetime import date, timedelta
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
|
||||
class UserMetricsService:
|
||||
""" Class to manage all the user info """
|
||||
|
||||
SERVICE_GEOCODER_NOKIA = 'geocoder_here'
|
||||
SERVICE_GEOCODER_GOOGLE = 'geocoder_google'
|
||||
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
|
||||
|
||||
GEOCODING_QUOTA_KEY = "geocoding_quota"
|
||||
GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit"
|
||||
|
||||
REDIS_CONNECTION_KEY = "redis_connection"
|
||||
REDIS_CONNECTION_HOST = "redis_host"
|
||||
REDIS_CONNECTION_PORT = "redis_port"
|
||||
REDIS_CONNECTION_DB = "redis_db"
|
||||
|
||||
def __init__(self, user_geocoder_config, redis_connection):
|
||||
self._user_geocoder_config = user_geocoder_config
|
||||
self._redis_connection = redis_connection
|
||||
self._username = user_geocoder_config.username
|
||||
self._orgname = user_geocoder_config.organization
|
||||
|
||||
def used_quota(self, service_type, date):
|
||||
""" Recover the used quota for the user in the current month """
|
||||
date_from, date_to = self.__current_billing_cycle()
|
||||
current_use = 0
|
||||
success_responses = self.get_metrics(service_type,
|
||||
'success_responses', date_from,
|
||||
date_to)
|
||||
empty_responses = self.get_metrics(service_type,
|
||||
'empty_responses', date_from,
|
||||
date_to)
|
||||
current_use += (success_responses + empty_responses)
|
||||
if service_type == self.SERVICE_GEOCODER_NOKIA:
|
||||
cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE,
|
||||
'success_responses', date_from,
|
||||
date_to)
|
||||
current_use += cache_hits
|
||||
|
||||
return current_use
|
||||
|
||||
def increment_service_use(self, service_type, metric, date=date.today(), amount=1):
|
||||
""" Increment the services uses in monthly and daily basis"""
|
||||
self.__increment_user_uses(service_type, metric, date, amount)
|
||||
if self._orgname:
|
||||
self.__increment_organization_uses(service_type, metric, date, amount)
|
||||
|
||||
def get_metrics(self, service, metric, date_from, date_to):
|
||||
aggregated_metric = 0
|
||||
key_prefix = "org" if self._orgname else "user"
|
||||
entity_name = self._orgname if self._orgname else self._username
|
||||
for date in self.__generate_date_range(date_from, date_to):
|
||||
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
|
||||
service, metric, date)
|
||||
score = self._redis_connection.zscore(redis_prefix, date.day)
|
||||
aggregated_metric += score if score else 0
|
||||
return aggregated_metric
|
||||
|
||||
# Private functions
|
||||
|
||||
def __increment_user_uses(self, service_type, metric, date, amount):
|
||||
redis_prefix = self.__parse_redis_prefix("user", self._username,
|
||||
service_type, metric, date)
|
||||
self._redis_connection.zincrby(redis_prefix, date.day, amount)
|
||||
|
||||
def __increment_organization_uses(self, service_type, metric, date, amount):
|
||||
redis_prefix = self.__parse_redis_prefix("org", self._orgname,
|
||||
service_type, metric, date)
|
||||
self._redis_connection.zincrby(redis_prefix, date.day, amount)
|
||||
|
||||
def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date):
|
||||
yearmonth_key = date.strftime('%Y%m')
|
||||
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
|
||||
service_type, metric,
|
||||
yearmonth_key)
|
||||
|
||||
return redis_name
|
||||
|
||||
def __current_billing_cycle(self):
|
||||
""" Return the begining and end date for the current billing cycle """
|
||||
end_period_day = self._user_geocoder_config.period_end_date.day
|
||||
today = date.today()
|
||||
if end_period_day > today.day:
|
||||
temp_date = today + relativedelta(months=-1)
|
||||
date_from = date(temp_date.year, temp_date.month, end_period_day)
|
||||
else:
|
||||
date_from = date(today.year, today.month, end_period_day)
|
||||
|
||||
return date_from, today
|
||||
|
||||
def __generate_date_range(self, date_from, date_to):
|
||||
for n in range(int((date_to - date_from).days + 1)):
|
||||
yield date_from + timedelta(n)
|
||||
@@ -0,0 +1 @@
|
||||
from redis_tools import RedisConnection
|
||||
@@ -0,0 +1,29 @@
|
||||
from redis.sentinel import Sentinel
|
||||
|
||||
|
||||
class RedisConnection:
|
||||
|
||||
REDIS_DEFAULT_USER_DB = 5
|
||||
REDIS_DEFAULT_TIMEOUT = 2 #seconds
|
||||
REDIS_SENTINEL_DEFAULT_PORT = 26379
|
||||
|
||||
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
|
||||
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
|
||||
self.sentinel_host = sentinel_host
|
||||
self.sentinel_port = sentinel_port
|
||||
self.sentinel_master_id = sentinel_master_id
|
||||
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
|
||||
self.redis_db = redis_db
|
||||
|
||||
def redis_connection(self):
|
||||
return self.__create_redis_connection()
|
||||
|
||||
def __create_redis_connection(self):
|
||||
sentinel = Sentinel([(self.sentinel_host,
|
||||
self.REDIS_SENTINEL_DEFAULT_PORT)],
|
||||
socket_timeout=self.timeout)
|
||||
return sentinel.master_for(
|
||||
self.sentinel_master_id,
|
||||
socket_timeout=self.timeout,
|
||||
db=self.redis_db
|
||||
)
|
||||
Reference in New Issue
Block a user