cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #107 from CartoDB/rename_and_metrics

Browse Source 
Rename and metrics
This commit is contained in:
Mario de Frutos
2016-03-17 10:23:14 +01:00
parent f85c4f1bbf f0a49d5fbc
commit 5d57624fd5
78 changed files with 3444 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -27,6 +27,7 @@ class ServiceConfig(object):
def organization(self):
return self._orgname
class RoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['username', 'orgname', 'mapzen_app_key']
@@ -34,11 +35,11 @@ class RoutingConfig(ServiceConfig):
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
def __init__(self, redis_connection, username, orgname=None,
mapzen_app_key=None):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, username,
orgname)
self._mapzen_app_key = mapzen_app_key
orgname)
db_config = ServicesDBConfig(db_conn)
self._mapzen_app_key = db_config.mapzen_routing_app_key
@property
def service_type(self):
@@ -53,7 +54,8 @@ class IsolinesRoutingConfig(ServiceConfig):
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
'period_end_date', 'username', 'orgname',
'heremaps_app_id', 'heremaps_app_code', 'geocoder_type']
'heremaps_app_id', 'heremaps_app_code',
'geocoder_type']
NOKIA_APP_ID_KEY = 'heremaps_app_id'
NOKIA_APP_CODE_KEY = 'heremaps_app_code'
QUOTA_KEY = 'here_isolines_quota'
@@ -64,24 +66,20 @@ class IsolinesRoutingConfig(ServiceConfig):
GEOCODER_TYPE_KEY = 'geocoder_type'
GOOGLE_GEOCODER = 'google'
def __init__(self, redis_connection, username, orgname=None,
heremaps_app_id=None, heremaps_app_code=None):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, username,
orgname)
config = self.__get_user_config(username, orgname, heremaps_app_id,
heremaps_app_code)
config = self.__get_user_config(username, orgname)
db_config = ServicesDBConfig(db_conn)
filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()}
self.__parse_config(filtered_config)
self.__parse_config(filtered_config, db_config)
def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
heremaps_app_code=None):
def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
user_config[self.NOKIA_APP_ID_KEY] = heremaps_app_id
user_config[self.NOKIA_APP_CODE_KEY] = heremaps_app_code
if orgname:
self.__get_organization_config(orgname, user_config)
@@ -96,7 +94,7 @@ class IsolinesRoutingConfig(ServiceConfig):
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
def __parse_config(self, filtered_config):
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower()
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
@@ -104,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig):
self._soft_isolines_limit = True
else:
self._soft_isolines_limit = False
self._heremaps_app_id = filtered_config[self.NOKIA_APP_ID_KEY]
self._heremaps_app_code = filtered_config[self.NOKIA_APP_CODE_KEY]
self._heremaps_app_id = db_config.heremaps_app_id
self._heremaps_app_code = db_config.heremaps_app_code
@property
def service_type(self):
@@ -138,14 +136,32 @@ class IsolinesRoutingConfig(ServiceConfig):
class InternalGeocoderConfig(ServiceConfig):
def __init__(self, redis_connection, username, orgname=None):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(InternalGeocoderConfig, self).__init__(redis_connection,
username, orgname)
db_config = ServicesDBConfig(db_conn)
self._log_path = db_config.geocoder_log_path
@property
def service_type(self):
return 'geocoder_internal'
@property
def is_high_resolution(self):
return False
@property
def cost_per_hit(self):
return 0
@property
def geocoding_quota(self):
return None
@property
def log_path(self):
return self._log_path
class GeocoderConfig(ServiceConfig):
@@ -154,8 +170,7 @@ class GeocoderConfig(ServiceConfig):
'geocoder_type', 'period_end_date',
'heremaps_app_id', 'heremaps_app_code', 'username',
'orgname']
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit',
'heremaps_app_id', 'heremaps_app_code']
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
NOKIA_GEOCODER = 'heremaps'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
@@ -169,25 +184,21 @@ class GeocoderConfig(ServiceConfig):
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, username, orgname=None,
heremaps_app_id=None, heremaps_app_code=None):
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(GeocoderConfig, self).__init__(redis_connection, username,
orgname)
config = self.__get_user_config(username, orgname, heremaps_app_id,
heremaps_app_code)
db_config = ServicesDBConfig(db_conn)
config = self.__get_user_config(username, orgname)
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
self.__parse_config(filtered_config, db_config)
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
heremaps_app_code=None):
def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id
user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code
if orgname:
self.__get_organization_config(orgname, user_config)
@@ -208,28 +219,29 @@ class GeocoderConfig(ServiceConfig):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]:
raise ConfigException("""Nokia geocoder configuration is missing. Check it please""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
return True
def __parse_config(self, filtered_config):
def __parse_config(self, filtered_config, db_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._log_path = db_config.geocoder_log_path
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_geocoding_limit = True
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY]
self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]
self._heremaps_app_id = db_config.heremaps_app_id
self._heremaps_app_code = db_config.heremaps_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
self._cost_per_hit = 0
@property
def service_type(self):
@@ -256,7 +268,10 @@ class GeocoderConfig(ServiceConfig):
@property
def geocoding_quota(self):
return self._geocoding_quota
if self.heremaps_geocoder:
return self._geocoding_quota
else:
return None
@property
def soft_geocoding_limit(self):
@@ -273,3 +288,82 @@ class GeocoderConfig(ServiceConfig):
@property
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def is_high_resolution(self):
return True
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def log_path(self):
return self._log_path
class ServicesDBConfig:
def __init__(self, db_conn):
self._db_conn = db_conn
return self._build()
def _build(self):
self._get_here_config()
self._get_mapzen_config()
self._get_logger_config()
def _get_here_config(self):
heremaps_conf_json = self._get_conf('heremaps_conf')
if not heremaps_conf_json:
raise ConfigException('Here maps configuration missing')
else:
heremaps_conf = json.loads(heremaps_conf_json)
self._heremaps_app_id = heremaps_conf['app_id']
self._heremaps_app_code = heremaps_conf['app_code']
self._heremaps_geocoder_cost_per_hit = heremaps_conf[
'geocoder_cost_per_hit']
def _get_mapzen_config(self):
mapzen_conf_json = self._get_conf('mapzen_conf')
if not mapzen_conf_json:
raise ConfigException('Mapzen configuration missing')
else:
mapzen_conf = json.loads(mapzen_conf_json)
self._mapzen_routing_app_key = mapzen_conf['routing_app_key']
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
logger_conf = json.loads(logger_conf_json)
self._geocoder_log_path = logger_conf['geocoder_log_path']
def _get_conf(self, key):
try:
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
conf = self._db_conn.execute(sql, 1)
return conf[0]['conf']
except:
raise ConfigException("Malformed config for {0}".format(key))
@property
def heremaps_app_id(self):
return self._heremaps_app_id
@property
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def heremaps_geocoder_cost_per_hit(self):
return self._heremaps_geocoder_cost_per_hit
@property
def mapzen_routing_app_key(self):
return self._mapzen_routing_app_key
@property
def geocoder_log_path(self):
return self._geocoder_log_path

74
server/lib/python/cartodb_services/cartodb_services/metrics/log.py Normal file
View File

@@ -0,0 +1,74 @@
from datetime import datetime
import abc
import json
import re
class LoggerFactory:
@classmethod
def build(self, service_config):
if re.match('geocoder_*', service_config.service_type):
return GeocoderLogger(service_config)
else:
return None
class Logger(object):
__metaclass__ = abc.ABCMeta
def __init__(self, file_path):
self._file_path = file_path
def dump_to_file(self, data):
with open(self._file_path, 'a') as logfile:
json.dump(data, logfile)
logfile.write('\n')
@abc.abstractproperty
def log(self, **data):
raise NotImplementedError('log method must be defined')
class GeocoderLogger(Logger):
def __init__(self, service_config):
super(GeocoderLogger, self).__init__(service_config.log_path)
self._service_config = service_config
def log(self, **data):
dump_data = self._dump_data(**data)
self.dump_to_file(dump_data)
def _dump_data(self, **data):
if data['success']:
cost = self._service_config.cost_per_hit
failed_rows = 0
successful_rows = 1
else:
cost = 0
failed_rows = 1
successful_rows = 0
if self._service_config.is_high_resolution:
kind = 'high-resolution'
else:
kind = 'internal'
return {
"batched": False,
"cache_hits": 0, # Always 0 because no cache involved
# https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211
"cost": cost,
"created_at": datetime.now().isoformat(),
"failed_rows": failed_rows,
"geocoder_type": self._service_config.service_type,
"kind": kind,
"processable_rows": 1,
"processed_rows": successful_rows,
"real_rows": successful_rows,
"success": data['success'],
"successful_rows": successful_rows,
"username": self._service_config.username,
"organization": self._service_config.organization
}

33
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -1,45 +1,43 @@
from user import UserMetricsService
from log import LoggerFactory
from datetime import date
import re
class QuotaService:
""" Class to manage all the quota operation for
the Geocoder SQL API Extension """
the Dataservices SQL API Extension """
def __init__(self, user_service_config, redis_connection):
self._user_service_config = user_service_config
# TODO First step to extract to a factory if needed in the future
self._quota_checker = QuotaChecker(user_service_config,
redis_connection)
self._user_service = UserMetricsService(
self._user_service_config, redis_connection)
self._user_service = UserMetricsService(self._user_service_config,
redis_connection)
self._logger = LoggerFactory.build(user_service_config)
def check_user_quota(self):
return self._quota_checker.check()
# TODO
# We are going to change this class to be the generic one and
# create specific for routing and geocoding services but because
# this implies change all the extension functions, we are going to
# make the change in a minor release
def increment_success_geocoder_use(self, amount=1):
def increment_success_service_use(self, amount=1):
self._user_service.increment_service_use(
self._user_service_config.service_type, "success_responses",
amount=amount)
self._log_service_process("success")
def increment_empty_geocoder_use(self, amount=1):
def increment_empty_service_use(self, amount=1):
self._user_service.increment_service_use(
self._user_service_config.service_type, "empty_responses",
amount=amount)
self._log_service_process("empty")
def increment_failed_geocoder_use(self, amount=1):
def increment_failed_service_use(self, amount=1):
self._user_service.increment_service_use(
self._user_service_config.service_type, "fail_responses",
amount=amount)
self._log_service_process("fail")
def increment_total_geocoder_use(self, amount=1):
def increment_total_service_use(self, amount=1):
self._user_service.increment_service_use(
self._user_service_config.service_type, "total_requests",
amount=amount)
@@ -49,6 +47,13 @@ class QuotaService:
self._user_service_config.service_type, "isolines_generated",
amount=amount)
def _log_service_process(self, event):
if self._logger:
if event is 'success' or event is 'empty':
self._logger.log(success=True)
elif event is 'empty':
self._logger.log(success=False)
class QuotaChecker:

2
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@@ -1,3 +1,3 @@
from redis_tools import RedisConnection
from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate
from polyline import PolyLine

87
server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py
View File

@@ -1,32 +1,87 @@
from redis.sentinel import Sentinel
from redis import StrictRedis
import json
class RedisConnection:
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds
def __init__(self, sentinel_master_id, redis_host, redis_port,
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.redis_host = redis_host
self.redis_port = redis_port
self.sentinel_master_id = sentinel_master_id
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def __init__(self, config):
self._config = config
def redis_connection(self):
return self.__create_redis_connection()
def __create_redis_connection(self):
if self.sentinel_master_id == None:
return StrictRedis(host=self.redis_host, port=self.redis_port, db=self.redis_db)
else:
sentinel = Sentinel([(self.redis_host,
self.redis_port)],
socket_timeout=self.timeout)
if self._config.sentinel_id:
sentinel = Sentinel([(self._config.host,
self._config.port)],
socket_timeout=self._config.timeout)
return sentinel.master_for(
self.sentinel_master_id,
socket_timeout=self.timeout,
db=self.redis_db
)
else:
conn = StrictRedis(host=self._config.host, port=self._config.port,
db=self._config.db)
return conn
class RedisDBConfig:
DEFAULT_USER_DB = 5
DEFAULT_TIMEOUT = 2 # seconds
def __init__(self, key, db_conn):
self._db_conn = db_conn
return self._build(key)
def _build(self, key):
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(
key)
conf = self._db_conn.execute(conf_query)[0]['conf']
if conf is None:
raise "There is no redis configuration defined"
else:
params = json.loads(conf)
self._host = params['redis_host']
self._port = params['redis_port']
if "timeout" in params:
self._timeout = params['timeout']
else:
self._timeout = self.DEFAULT_TIMEOUT
if "redis_db" in params:
self._db = params['redis_db']
else:
self._db = self.DEFAULT_USER_DB
if "sentinel_master_id" in params:
self._sentinel_id = params["sentinel_master_id"]
else:
self._sentinel_id = None
def __str__(self):
return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format(
self.host, self.port, self.sentinel_id, self.db)
@property
def host(self):
return self._host
@property
def port(self):
return self._port
@property
def timeout(self):
return self._timeout
@property
def db(self):
return self._db
@property
def sentinel_id(self):
return self._sentinel_id

2
server/lib/python/cartodb_services/setup.py
View File

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.3.1',
version='0.3.2',
description='CartoDB Services API Python Library',

18
server/lib/python/cartodb_services/test/test_config.py
View File

@@ -10,12 +10,12 @@ class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
self.plpy_mock = test_helper.build_plpy_mock()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
'test_user', None)
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit is False
@@ -25,17 +25,17 @@ class TestConfig(TestCase):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
'test_user', 'test_org')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
def test_should_raise_exception_when_missing_parameters(self):
plpy_mock = test_helper.build_plpy_mock(empty=True)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)
self.redis_conn, plpy_mock, 'test_user',
None)

18
server/lib/python/cartodb_services/test/test_helper.py
View File

@@ -1,4 +1,5 @@
from datetime import datetime, date
from mock import Mock
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@@ -31,3 +32,20 @@ def increment_geocoder_uses(redis_conn, username, orgname=None,
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service, metric, yearmonth)
redis_conn.zincrby(redis_name, date.day, amount)
def build_plpy_mock(empty=False):
plpy_mock = Mock()
if not empty:
plpy_mock.execute.side_effect = _plpy_execute_side_effect
return plpy_mock
def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing_app_key": "app_key"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]

14
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@@ -63,18 +63,18 @@ class TestQuotaService(TestCase):
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
qs.increment_success_service_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
qs.increment_success_service_use()
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
qs.increment_success_service_use(amount=2)
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
@@ -88,9 +88,9 @@ class TestQuotaService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
plpy_mock = test_helper.build_plpy_mock()
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

7
server/lib/python/cartodb_services/test/test_user_service.py
View File

@@ -4,6 +4,7 @@ from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from mock import Mock
from nose.tools import assert_raises
from datetime import timedelta
@@ -84,7 +85,7 @@ class TestUserService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
plpy_mock = test_helper.build_plpy_mock()
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname,)
return UserMetricsService(geocoder_config, self.redis_conn)