cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'development' into 346-user-rate-limits

Browse Source 
# Conflicts:
#	README.md
#	server/extension/sql/20_geocode_street.sql
#	server/lib/python/cartodb_services/cartodb_services/metrics/config.py
This commit is contained in:
Javier Goizueta
2017-03-14 19:00:53 +01:00
parent 6c5ca97468 a64557b50e
commit 05b29967c7
30 changed files with 3916 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
+9 -6
View File
@@ -52,14 +52,17 @@ class HereMapsGeocoder(Traceable):
'strictlanguagemode'
] + ADDRESS_PARAMS
def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS):
service_params = service_params or {}
self.app_id = app_id
self.app_code = app_code
self._logger = logger
self.maxresults = maxresults
self.gen = gen
self.host = host
self.gen = service_params.get('gen', self.DEFAULT_GEN)
self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL)
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
def geocode(self, **kwargs):
params = {}
@@ -92,9 +95,9 @@ class HereMapsGeocoder(Traceable):
request_params.update(params)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self.host, HTTPAdapter(max_retries=self.max_retries))
response = session.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
timeout=(self.connect_timeout, self.read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
server/lib/python/cartodb_services/cartodb_services/here/routing.py
+10 -5
View File
@@ -30,12 +30,17 @@ class HereMapsRoutingIsoline(Traceable):
'quality'
]
def __init__(self, app_id, app_code, logger,
base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_id, app_code, logger, service_params=None):
service_params = service_params or {}
self._app_id = app_id
self._app_code = app_code
self._logger = logger
self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH)
base_url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
isoline_path = service_params.get('isoline_path', self.ISOLINE_PATH)
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
self._url = "{0}{1}".format(base_url, isoline_path)
def calculate_isodistance(self, source, mode, data_range, options=[]):
return self.__calculate_isolines(source, mode, data_range, 'distance',
@@ -57,9 +62,9 @@ class HereMapsRoutingIsoline(Traceable):
parsed_options)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self._url, HTTPAdapter(max_retries=self.max_retries))
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
timeout=(self.connect_timeout, self.read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text)
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
+13 -4
View File
@@ -17,23 +17,32 @@ class MapzenGeocoder(Traceable):
CONNECT_TIMEOUT = 10
MAX_RETRIES = 1
def __init__(self, app_key, logger, base_url=BASE_URL):
def __init__(self, app_key, logger, service_params=None):
service_params = service_params or {}
self._app_key = app_key
self._url = base_url
self._url = service_params.get('base_url', self.BASE_URL)
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
self._logger = logger
@qps_retry(qps=20)
def geocode(self, searchtext, city=None, state_province=None,
country=None, search_type=None):
# Remove the search_type if its address from the params sent to mapzen
if search_type and search_type.lower() == 'address':
search_type = None
request_params = self._build_requests_parameters(searchtext, city,
state_province,
country, search_type)
try:
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
timeout=(self._connect_timeout, self._read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)
server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py
+10 -5
View File
@@ -20,10 +20,15 @@ class MapzenIsochrones:
"car": "auto"
}
def __init__(self, app_key, logger, base_url=BASE_URL):
def __init__(self, app_key, logger, service_params=None):
service_params = service_params or {}
self._app_key = app_key
self._url = base_url
self._logger = logger
self._url = service_params.get('base_url', self.BASE_URL)
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
@qps_retry(qps=7)
def isochrone(self, locations, costing, ranges):
@@ -32,10 +37,10 @@ class MapzenIsochrones:
try:
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT,
self.READ_TIMEOUT))
timeout=(self._connect_timeout,
self._read_timeout))
if response.status_code is requests.codes.ok:
return self._parse_response(response)
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
+8 -3
View File
@@ -23,9 +23,14 @@ class MatrixClient(Traceable):
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
def __init__(self, matrix_key, logger):
def __init__(self, matrix_key, logger, service_params=None):
service_params = service_params or {}
self._matrix_key = matrix_key
self._logger = logger
self._url = service_params.get('one_to_many_url', self.ONE_TO_MANY_URL)
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
"""Get distances and times to a set of locations.
See https://mapzen.com/documentation/matrix/api-reference/
@@ -44,8 +49,8 @@ class MatrixClient(Traceable):
'costing': costing,
'api_key': self._matrix_key
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
response = requests.get(self._url, params=request_params,
timeout=(self._connect_timeout, self._read_timeout))
self.add_response_data(response, self._logger)
if response.status_code != requests.codes.ok:
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
+8 -4
View File
@@ -33,10 +33,14 @@ class MapzenRouting(Traceable):
METRICS_UNITS = 'kilometers'
IMPERIAL_UNITS = 'miles'
def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_key, logger, service_params=None):
service_params = service_params or {}
self._app_key = app_key
self._url = base_url
self._logger = logger
self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL)
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
@qps_retry
def calculate_route_point_to_point(self, waypoints, mode,
@@ -50,9 +54,9 @@ class MapzenRouting(Traceable):
request_params = self.__parse_request_parameters(json_request_params)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
timeout=(self._connect_timeout, self._read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_routing_response(response.text)
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
+64
View File
@@ -154,6 +154,7 @@ class RoutingConfig(ServiceConfig):
if not self._routing_provider:
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
self._set_monthly_quota()
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@@ -171,6 +172,10 @@ class RoutingConfig(ServiceConfig):
def mapzen_api_key(self):
return self._mapzen_api_key
@property
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def monthly_quota(self):
return self._monthly_quota
@@ -229,8 +234,11 @@ class IsolinesRoutingConfig(ServiceConfig):
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._heremaps_app_id = db_config.heremaps_isolines_app_id
self._heremaps_app_code = db_config.heremaps_isolines_app_code
self._heremaps_service_params = db_config.heremaps_isolines_service_params
elif self._isolines_provider == self.MAPZEN_PROVIDER:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
@property
def service_type(self):
@@ -263,10 +271,22 @@ class IsolinesRoutingConfig(ServiceConfig):
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def heremaps_service_params(self):
return self._heremaps_service_params
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@property
def mapzen_matrix_service_params(self):
return self._mapzen_matrix_service_params
@property
def mapzen_isochrones_service_params(self):
return self._mapzen_isochrones_service_params
@property
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@@ -375,6 +395,7 @@ class GeocoderConfig(ServiceConfig):
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
self._heremaps_service_params = db_config.heremaps_geocoder_service_params
elif self._geocoder_provider == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
@@ -382,6 +403,7 @@ class GeocoderConfig(ServiceConfig):
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._cost_per_hit = 0
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
self._rate_limit = self._get_rate_limit('geocoder')
@@ -437,10 +459,18 @@ class GeocoderConfig(ServiceConfig):
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def heremaps_service_params(self):
return self._heremaps_service_params
@property
def mapzen_api_key(self):
return self._mapzen_api_key
@property
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def is_high_resolution(self):
return True
@@ -457,6 +487,10 @@ class GeocoderConfig(ServiceConfig):
def rate_limit(self):
return self._rate_limit
@property
def service(self):
return self._service
class ServicesDBConfig:
def __init__(self, db_conn, username, orgname):
@@ -493,8 +527,10 @@ class ServicesDBConfig:
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
'geocoder_cost_per_hit']
self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
def _get_mapzen_config(self):
mapzen_conf_json = self._get_conf('mapzen_conf')
@@ -504,10 +540,14 @@ class ServicesDBConfig:
mapzen_conf = json.loads(mapzen_conf_json)
self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
@@ -547,6 +587,10 @@ class ServicesDBConfig:
def heremaps_isolines_app_code(self):
return self._heremaps_isolines_app_code
@property
def heremaps_isolines_service_params(self):
return self._heremaps_isolines_service_params
@property
def heremaps_geocoder_app_id(self):
return self._heremaps_geocoder_app_id
@@ -559,6 +603,10 @@ class ServicesDBConfig:
def heremaps_geocoder_cost_per_hit(self):
return self._heremaps_geocoder_cost_per_hit
@property
def heremaps_geocoder_service_params(self):
return self._heremaps_geocoder_service_params
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@@ -567,6 +615,14 @@ class ServicesDBConfig:
def mapzen_matrix_monthly_quota(self):
return self._mapzen_matrix_quota
@property
def mapzen_matrix_service_params(self):
return self._mapzen_matrix_service_params
@property
def mapzen_isochrones_service_params(self):
return self._mapzen_isochrones_service_params
@property
def mapzen_routing_api_key(self):
return self._mapzen_routing_api_key
@@ -575,6 +631,10 @@ class ServicesDBConfig:
def mapzen_routing_monthly_quota(self):
return self._mapzen_routing_quota
@property
def mapzen_routing_service_params(self):
return self._mapzen_routing_service_params
@property
def mapzen_geocoder_api_key(self):
return self._mapzen_geocoder_api_key
@@ -583,6 +643,10 @@ class ServicesDBConfig:
def mapzen_geocoder_monthly_quota(self):
return self._mapzen_geocoder_quota
@property
def mapzen_geocoder_service_params(self):
return self._mapzen_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
+2 -5
View File
@@ -118,12 +118,9 @@ class UserMetricsService:
for date in self.__generate_date_range(date_from, date_to):
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
service, metric, date)
score = self._redis_connection.zscore(redis_prefix, date.day)
aggregated_metric += int(score) if score else 0
zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
if str(date.day) != zero_padded_day:
score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
aggregated_metric += int(score) if score else 0
score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
aggregated_metric += int(score) if score else 0
return aggregated_metric
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py
+10 -2
View File
@@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object):
log_path,
mapzen_api_key,
username,
organization):
organization,
service_params):
self._geocoding_quota = geocoding_quota
self._soft_geocoding_limit = soft_geocoding_limit
self._period_end_date = period_end_date
@@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object):
self._mapzen_api_key = mapzen_api_key
self._username = username
self._organization = organization
self._service_params = service_params
# Kind of generic properties. Note which ones are for actually running the
# service and which ones are needed for quota stuff.
@@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object):
def organization(self):
return self._organization
@property
def service_params(self):
return self._service_params
# TODO: for BW compat, remove
@property
def google_geocoder(self):
@@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object):
def get(self):
mapzen_server_conf = self._server_conf.get('mapzen_conf')
mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {})
geocoding_quota = self._get_quota(mapzen_server_conf)
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
@@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object):
log_path,
mapzen_api_key,
self._username,
self._orgname)
self._orgname,
mapzen_service_params)
def _get_quota(self, mapzen_server_conf):
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')