Solving errors and refactor

This commit is contained in:
Antonio
2018-01-04 15:00:52 +01:00
parent 51d97228dc
commit e7c0628751
20 changed files with 60 additions and 74 deletions
@@ -2,4 +2,3 @@ from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder from geocoder import MapboxGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse from isolines import MapboxIsolines, MapboxIsochronesResponse
from matrix_client import MapboxMatrixClient from matrix_client import MapboxMatrixClient
from exceptions import ServiceException
@@ -1,18 +0,0 @@
'''
Exceptions for the Mapbox services Python wrapper.
'''
class ServiceException(Exception):
'''
Exception to be raised if any Service problem is found.
'''
def __init__(self, code, message):
self.code = code
self.message = message
def __str__(self):
return repr('ServiceException ({code}): {message}'.format(
code=self.code,
message=self.message))
@@ -6,7 +6,7 @@ import json
import requests import requests
from mapbox import Geocoder from mapbox import Geocoder
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
from cartodb_services.mapbox.exceptions import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
GEOCODER_NAME = 'geocoder_name' GEOCODER_NAME = 'geocoder_name'
@@ -69,4 +69,4 @@ class MapboxGeocoder(Traceable):
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return self._parse_geocoder_response(response.text) return self._parse_geocoder_response(response.text)
else: else:
raise ServiceException(response.status_code, response.content) raise ServiceException(response.status_code, response)
@@ -6,7 +6,7 @@ import requests
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
from cartodb_services.tools.coordinates import (validate_coordinates, from cartodb_services.tools.coordinates import (validate_coordinates,
marshall_coordinates) marshall_coordinates)
from exceptions import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/' BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/'
@@ -69,4 +69,4 @@ class MapboxMatrixClient(Traceable):
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return response.text return response.text
else: else:
raise ServiceException(response.status_code, response.content) raise ServiceException(response.status_code, response)
@@ -8,7 +8,7 @@ from cartodb_services.metrics import Traceable
from cartodb_services.tools import PolyLine from cartodb_services.tools import PolyLine
from cartodb_services.tools.coordinates import (validate_coordinates, from cartodb_services.tools.coordinates import (validate_coordinates,
marshall_coordinates) marshall_coordinates)
from cartodb_services.mapbox.exceptions import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
BASEURI = ('https://api.mapbox.com/directions/v5/mapbox/{profile}/' BASEURI = ('https://api.mapbox.com/directions/v5/mapbox/{profile}/'
@@ -86,7 +86,7 @@ class MapboxRouting(Traceable):
elif response.status_code == requests.codes.bad_request: elif response.status_code == requests.codes.bad_request:
return MapboxRoutingResponse(None, None, None) return MapboxRoutingResponse(None, None, None)
else: else:
raise ServiceException(response.status_code, response.content) raise ServiceException(response.status_code, response)
class MapboxRoutingResponse: class MapboxRoutingResponse:
@@ -1,33 +0,0 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class WrongParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Wrong parameters passed: ' + json.dumps(self.value))
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')
class TimeoutException(Exception):
def __str__(self):
return repr('Timeout requesting to mapzen server')
class ServiceException(Exception):
def __init__(self, message, response):
self.message = message
self.response = response
def response(self):
return self.response
def __str__(self):
return self.message
@@ -3,7 +3,9 @@ import json
import re import re
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from cartodb_services.tools.exceptions import (WrongParams,
MalformedResult,
ServiceException)
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
@@ -3,7 +3,9 @@ import json
import re import re
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from cartodb_services.tools.exceptions import (WrongParams,
MalformedResult,
ServiceException)
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
@@ -1,7 +1,7 @@
import requests import requests
import json import json
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
from exceptions import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
@@ -3,7 +3,9 @@ import json
import re import re
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from cartodb_services.tools.exceptions import (WrongParams,
MalformedResult,
ServiceException)
from cartodb_services.tools.qps import qps_retry from cartodb_services.tools.qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import MetricsDataGatherer, Traceable from cartodb_services.metrics import MetricsDataGatherer, Traceable
@@ -4,4 +4,4 @@ from polyline import PolyLine
from log import Logger, LoggerConfig from log import Logger, LoggerConfig
from rate_limiter import RateLimiter from rate_limiter import RateLimiter
from service_manager import ServiceManager, RateLimitExceeded from service_manager import ServiceManager, RateLimitExceeded
from legacy_service_manager import LegacyServiceManager from legacy_service_manager import LegacyServiceManager
@@ -1,3 +1,31 @@
import json
class TimeoutException(Exception): class TimeoutException(Exception):
def __str__(self): def __str__(self):
return repr('Timeout requesting to server') return repr('Timeout requesting to server')
class ServiceException(Exception):
def __init__(self, message, response):
self.message = message
self.response = response
def response(self):
return self.response
def __str__(self):
return self.message
class WrongParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Wrong parameters passed: ' + json.dumps(self.value))
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')
@@ -6,7 +6,8 @@ from exceptions import TimeoutException
DEFAULT_RETRY_TIMEOUT = 60 DEFAULT_RETRY_TIMEOUT = 60
DEFAULT_QUERIES_PER_SECOND = 10 DEFAULT_QUERIES_PER_SECOND = 10
def qps_retry(original_function=None,**options):
def qps_retry(original_function=None, **options):
""" Query Per Second retry decorator """ Query Per Second retry decorator
The intention of this decorator is to retry requests against third The intention of this decorator is to retry requests against third
party services that has QPS restriction. party services that has QPS restriction.
@@ -55,7 +56,7 @@ class QPSService:
def retry(self, first_request_time, retry_count): def retry(self, first_request_time, retry_count):
elapsed = datetime.now() - first_request_time elapsed = datetime.now() - first_request_time
if elapsed.total_seconds() > self._retry_timeout: if elapsed.total_seconds() > self._retry_timeout:
raise TimeoutException() raise TimeoutException()
# inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
@@ -1,7 +1,7 @@
import unittest import unittest
from mock import Mock from mock import Mock
from cartodb_services.mapbox import MapboxGeocoder from cartodb_services.mapbox import MapboxGeocoder
from cartodb_services.mapbox import ServiceException from cartodb_services.tools.exceptions import ServiceException
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg' VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
INVALID_TOKEN = 'invalid_token' INVALID_TOKEN = 'invalid_token'
@@ -2,7 +2,7 @@ import unittest
from mock import Mock from mock import Mock
from cartodb_services.mapbox import MapboxMatrixClient from cartodb_services.mapbox import MapboxMatrixClient
from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE
from cartodb_services.mapbox import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg' VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
@@ -2,7 +2,7 @@ import unittest
from mock import Mock from mock import Mock
from cartodb_services.mapbox import MapboxRouting from cartodb_services.mapbox import MapboxRouting
from cartodb_services.mapbox.routing import DEFAULT_PROFILE from cartodb_services.mapbox.routing import DEFAULT_PROFILE
from cartodb_services.mapbox import ServiceException from cartodb_services.tools.exceptions import ServiceException
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg' VALID_TOKEN = 'pk.eyJ1IjoiYWNhcmxvbiIsImEiOiJjamJuZjQ1Zjc0Ymt4Mnh0YmFrMmhtYnY4In0.gt9cw0VeKc3rM2mV5pcEmg'
@@ -6,7 +6,8 @@ import requests_mock
from mock import Mock from mock import Mock
from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.exceptions import MalformedResult, TimeoutException from cartodb_services.tools.exceptions import (MalformedResult,
TimeoutException)
requests_mock.Mocker.TEST_PREFIX = 'test_' requests_mock.Mocker.TEST_PREFIX = 'test_'
@@ -4,7 +4,7 @@ import requests_mock
from mock import Mock from mock import Mock
from cartodb_services.mapzen import MapzenIsochrones from cartodb_services.mapzen import MapzenIsochrones
from cartodb_services.mapzen.exceptions import ServiceException from cartodb_services.tools.exceptions import ServiceException
requests_mock.Mocker.TEST_PREFIX = 'test_' requests_mock.Mocker.TEST_PREFIX = 'test_'
@@ -9,7 +9,7 @@ from urlparse import urlparse, parse_qs
from mock import Mock from mock import Mock
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.exceptions import WrongParams from cartodb_services.tools.exceptions import WrongParams
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
requests_mock.Mocker.TEST_PREFIX = 'test_' requests_mock.Mocker.TEST_PREFIX = 'test_'
@@ -2,13 +2,15 @@ import requests
from unittest import TestCase from unittest import TestCase
from nose.tools import assert_raises from nose.tools import assert_raises
from datetime import datetime, date from datetime import datetime, date
from cartodb_services.mapzen.qps import qps_retry from cartodb_services.tools.qps import qps_retry
from cartodb_services.mapzen.exceptions import ServiceException, TimeoutException from cartodb_services.tools.exceptions import (ServiceException,
TimeoutException)
import requests_mock import requests_mock
import mock import mock
requests_mock.Mocker.TEST_PREFIX = 'test_' requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker() @requests_mock.Mocker()
class TestQPS(TestCase): class TestQPS(TestCase):
QPS_ERROR_MESSAGE = "Queries per second exceeded: Queries exceeded (10 allowed)" QPS_ERROR_MESSAGE = "Queries per second exceeded: Queries exceeded (10 allowed)"
@@ -26,7 +28,7 @@ class TestQPS(TestCase):
return self.QPS_ERROR_MESSAGE return self.QPS_ERROR_MESSAGE
req_mock.register_uri('GET', 'http://localhost/test_qps', req_mock.register_uri('GET', 'http://localhost/test_qps',
text=_text_cb) text=_text_cb)
with self.assertRaises(TimeoutException): with self.assertRaises(TimeoutException):
c = TestClass() c = TestClass()
c.test() c.test()