Refactor python library to unify and rename as cdb_services

2016-02-03 10:21:57 +01:00
parent e7c58b9a51
commit 3960c13484
35 changed files with 397 additions and 872 deletions
@@ -0,0 +1,3 @@
+from config import GeocoderConfig, ConfigException
+from quota import QuotaService
+from user import UserMetricsService
@@ -0,0 +1,148 @@
+import json
+from dateutil.parser import parse as date_parse
+
+
+class ConfigException(Exception):
+    pass
+
+
+class GeocoderConfig:
+
+    GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
+                            'geocoding_quota', 'soft_geocoding_limit',
+                            'geocoder_type', 'period_end_date',
+                            'heremaps_app_id', 'heremaps_app_code', 'username',
+                            'orgname']
+    NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit',
+                                     'heremaps_app_id', 'heremaps_app_code']
+    NOKIA_GEOCODER = 'heremaps'
+    NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
+    NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
+    GOOGLE_GEOCODER = 'google'
+    GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
+    GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
+    GEOCODER_TYPE = 'geocoder_type'
+    QUOTA_KEY = 'geocoding_quota'
+    SOFT_LIMIT_KEY = 'soft_geocoding_limit'
+    USERNAME_KEY = 'username'
+    ORGNAME_KEY = 'orgname'
+    PERIOD_END_DATE = 'period_end_date'
+
+    def __init__(self, redis_connection, username, orgname=None,
+                 heremaps_app_id=None, heremaps_app_code=None):
+        self._redis_connection = redis_connection
+        config = self.__get_user_config(username, orgname, heremaps_app_id,
+                                        heremaps_app_code)
+        filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
+        self.__check_config(filtered_config)
+        self.__parse_config(filtered_config)
+
+    def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
+                          heremaps_app_code=None):
+        user_config = self._redis_connection.hgetall(
+            "rails:users:{0}".format(username))
+        if not user_config:
+            raise ConfigException("""There is no user config available. Please check your configuration.'""")
+        else:
+            user_config[self.USERNAME_KEY] = username
+            user_config[self.ORGNAME_KEY] = orgname
+            user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id
+            user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code
+            if orgname:
+                self.__get_organization_config(orgname, user_config)
+
+            return user_config
+
+    def __get_organization_config(self, orgname, user_config):
+        org_config = self._redis_connection.hgetall(
+            "rails:orgs:{0}".format(orgname))
+        if not org_config:
+            raise ConfigException("""There is no organization config available. Please check your configuration.'""")
+        else:
+            user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
+            user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
+            user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
+            user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
+
+    def __check_config(self, filtered_config):
+        if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
+            if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
+                raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
+            if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]:
+                raise ConfigException("""Nokia geocoder configuration is missing. Check it please""")
+        elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
+            if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
+                raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
+
+        return True
+
+    def __parse_config(self, filtered_config):
+        self._username = filtered_config[self.USERNAME_KEY].lower()
+        if filtered_config[self.ORGNAME_KEY]:
+            self._orgname = filtered_config[self.ORGNAME_KEY].lower()
+        else:
+            self._orgname = None
+        self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
+        self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
+        self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
+        if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
+            self._soft_geocoding_limit = True
+        else:
+            self._soft_geocoding_limit = False
+        if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
+            self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY]
+            self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]
+        elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
+            self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
+            self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
+
+    @property
+    def service_type(self):
+        if self._geocoder_type == self.GOOGLE_GEOCODER:
+            return 'geocoder_google'
+        else:
+            return 'geocoder_here'
+
+    @property
+    def heremaps_geocoder(self):
+        return self._geocoder_type == self.NOKIA_GEOCODER
+
+    @property
+    def google_geocoder(self):
+        return self._geocoder_type == self.GOOGLE_GEOCODER
+
+    @property
+    def google_client_id(self):
+        return self._google_maps_client_id
+
+    @property
+    def google_api_key(self):
+        return self._google_maps_api_key
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def heremaps_app_id(self):
+        return self._heremaps_app_id
+
+    @property
+    def heremaps_app_code(self):
+        return self._heremaps_app_code
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._orgname
@@ -0,0 +1,49 @@
+from user import UserMetricsService
+from datetime import date
+
+
+class QuotaService:
+    """ Class to manage all the quota operation for
+    the Geocoder SQL API Extension """
+
+    def __init__(self, user_geocoder_config, redis_connection):
+        self._user_geocoder_config = user_geocoder_config
+        self._user_service = UserMetricsService(
+            self._user_geocoder_config, redis_connection)
+
+    def check_user_quota(self):
+        """ Check if the current user quota surpasses the current quota """
+        # We don't have quota check for google geocoder
+        if self._user_geocoder_config.google_geocoder:
+            return True
+
+        user_quota = self._user_geocoder_config.geocoding_quota
+        today = date.today()
+        service_type = self._user_geocoder_config.service_type
+        current_used = self._user_service.used_quota(service_type, today)
+        soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit
+
+        if soft_geocoding_limit or current_used <= user_quota:
+            return True
+        else:
+            return False
+
+    def increment_success_geocoder_use(self, amount=1):
+        self._user_service.increment_service_use(
+            self._user_geocoder_config.service_type, "success_responses",
+            amount=amount)
+
+    def increment_empty_geocoder_use(self, amount=1):
+        self._user_service.increment_service_use(
+            self._user_geocoder_config.service_type, "empty_responses",
+            amount=amount)
+
+    def increment_failed_geocoder_use(self, amount=1):
+        self._user_service.increment_service_use(
+            self._user_geocoder_config.service_type, "fail_responses",
+            amount=amount)
+
+    def increment_total_geocoder_use(self, amount=1):
+        self._user_service.increment_service_use(
+            self._user_geocoder_config.service_type, "total_requests",
+            amount=amount)
@@ -0,0 +1,96 @@
+from datetime import date, timedelta
+from dateutil.relativedelta import relativedelta
+
+
+class UserMetricsService:
+    """ Class to manage all the user info """
+
+    SERVICE_GEOCODER_NOKIA = 'geocoder_here'
+    SERVICE_GEOCODER_GOOGLE = 'geocoder_google'
+    SERVICE_GEOCODER_CACHE = 'geocoder_cache'
+
+    GEOCODING_QUOTA_KEY = "geocoding_quota"
+    GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit"
+
+    REDIS_CONNECTION_KEY = "redis_connection"
+    REDIS_CONNECTION_HOST = "redis_host"
+    REDIS_CONNECTION_PORT = "redis_port"
+    REDIS_CONNECTION_DB = "redis_db"
+
+    def __init__(self, user_geocoder_config, redis_connection):
+        self._user_geocoder_config = user_geocoder_config
+        self._redis_connection = redis_connection
+        self._username = user_geocoder_config.username
+        self._orgname = user_geocoder_config.organization
+
+    def used_quota(self, service_type, date):
+        """ Recover the used quota for the user in the current month """
+        date_from, date_to = self.__current_billing_cycle()
+        current_use = 0
+        success_responses = self.get_metrics(service_type,
+                                             'success_responses', date_from,
+                                             date_to)
+        empty_responses = self.get_metrics(service_type,
+                                           'empty_responses', date_from,
+                                           date_to)
+        current_use += (success_responses + empty_responses)
+        if service_type == self.SERVICE_GEOCODER_NOKIA:
+            cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE,
+                                          'success_responses', date_from,
+                                          date_to)
+            current_use += cache_hits
+
+        return current_use
+
+    def increment_service_use(self, service_type, metric, date=date.today(), amount=1):
+        """ Increment the services uses in monthly and daily basis"""
+        self.__increment_user_uses(service_type, metric, date, amount)
+        if self._orgname:
+            self.__increment_organization_uses(service_type, metric, date, amount)
+
+    def get_metrics(self, service, metric, date_from, date_to):
+        aggregated_metric = 0
+        key_prefix = "org" if self._orgname else "user"
+        entity_name = self._orgname if self._orgname else self._username
+        for date in self.__generate_date_range(date_from, date_to):
+            redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
+                                                     service, metric, date)
+            score = self._redis_connection.zscore(redis_prefix, date.day)
+            aggregated_metric += score if score else 0
+        return aggregated_metric
+
+    # Private functions
+
+    def __increment_user_uses(self, service_type, metric, date, amount):
+        redis_prefix = self.__parse_redis_prefix("user", self._username,
+                                                 service_type, metric, date)
+        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+
+    def __increment_organization_uses(self, service_type, metric, date, amount):
+        redis_prefix = self.__parse_redis_prefix("org", self._orgname,
+                                                 service_type, metric, date)
+        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+
+    def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date):
+        yearmonth_key = date.strftime('%Y%m')
+        redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
+                                              service_type, metric,
+                                              yearmonth_key)
+
+        return redis_name
+
+    def __current_billing_cycle(self):
+        """ Return the begining and end date for the current billing cycle """
+        end_period_day = self._user_geocoder_config.period_end_date.day
+        today = date.today()
+        if end_period_day > today.day:
+            temp_date = today + relativedelta(months=-1)
+            date_from = date(temp_date.year, temp_date.month, end_period_day)
+        else:
+            date_from = date(today.year, today.month, end_period_day)
+
+        return date_from, today
+
+    def __generate_date_range(self, date_from, date_to):
+        for n in range(int((date_to - date_from).days + 1)):
+            yield date_from + timedelta(n)