Merging changes

This commit is contained in:
Antonio
2018-03-16 14:30:01 +01:00
17 changed files with 7059 additions and 26 deletions

22
NEWS.md
View File

@@ -1,4 +1,26 @@
March 16th, 2018
================
* Version `0.30.4` of server side
* Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries
March 16th, 2018
================
* Version `0.30.3` of server side
* Fix problem with invalid Mapbox isolines
March 14th, 2018
================
* Version `0.17.4` of the python library
* Fix bug with previous version when checking quotas
* Version `0.17.3` of the python library
* Fix bug with Mapbox routing not using the proper quota value
February 22th, 2018
==================
* Version `0.17.2` of the python library
* Fix bug with Mapbox isolines not stopping at the seacoast
February 27th, 2018
==================
* Version `0.17.1` of the python library

View File

@@ -276,7 +276,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None

View File

@@ -1,6 +1,6 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(TEXT, TEXT, geometry(Point, 4326)[], TEXT);

View File

@@ -3190,7 +3190,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
@@ -3254,7 +3254,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None

View File

@@ -0,0 +1,70 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

View File

@@ -0,0 +1,70 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

View File

@@ -0,0 +1,69 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,69 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
username TEXT,
orgname TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.tools import ServiceManager
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
from cartodb_services.tools import Coordinate
from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
service_manager.assert_within_limits()
try:
client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
mapbox_isolines = MapboxIsolines(client, service_manager.logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = Coordinate(lon,lat)
else:
raise Exception('source is NULL')
profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
for r in data_range:
if len(isolines[r]) >= 3:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
result.append([source, r, multipolygon])
service_manager.quota_service.increment_success_service_use()
service_manager.quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to get Mapbox isolines')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

File diff suppressed because it is too large Load Diff

View File

@@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None
@@ -233,7 +233,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
else:
multipolygon = None

View File

@@ -4,6 +4,7 @@ Uses the Mapbox Time Matrix service.
'''
import json
from cartodb_services.tools import Coordinate
from cartodb_services.tools.spherical import (get_angles,
calculate_dest_location)
from cartodb_services.mapbox.matrix_client import (validate_profile,
@@ -11,7 +12,9 @@ from cartodb_services.mapbox.matrix_client import (validate_profile,
PROFILE_WALKING,
PROFILE_DRIVING,
PROFILE_CYCLING,
ENTRY_DURATIONS)
ENTRY_DURATIONS,
ENTRY_DESTINATIONS,
ENTRY_LOCATION)
MAX_SPEEDS = {
PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed
@@ -53,6 +56,7 @@ class MapboxIsolines():
return []
costs = [None] * number_of_angles
destinations = [None] * number_of_angles
for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
if cost:
@@ -60,7 +64,11 @@ class MapboxIsolines():
else:
costs[idx] = isorange
return costs
for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
destination[ENTRY_LOCATION][1])
return costs, destinations
def calculate_isochrone(self, origin, time_ranges,
profile=DEFAULT_PROFILE):
@@ -122,10 +130,12 @@ class MapboxIsolines():
# NOTE: sometimes it cannot calculate the cost and returns None.
# Just assume isorange and stop the calculations there
costs = cost_method(origin=origin, targets=location_estimates,
isorange=isorange, profile=profile,
unit_factor=unit_factor,
number_of_angles=number_of_angles)
costs, destinations = cost_method(origin=origin,
targets=location_estimates,
isorange=isorange,
profile=profile,
unit_factor=unit_factor,
number_of_angles=number_of_angles)
if not costs:
continue
@@ -152,8 +162,8 @@ class MapboxIsolines():
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c != isorange:
location_estimates_filtered.append(location_estimates[i])
if c != isorange and c < isorange * (1 + tolerance):
location_estimates_filtered.append(destinations[i])
return location_estimates_filtered

View File

@@ -30,6 +30,8 @@ VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
PROFILE_WALKING]
ENTRY_DURATIONS = 'durations'
ENTRY_DESTINATIONS = 'destinations'
ENTRY_LOCATION = 'location'
def validate_profile(profile):

View File

@@ -157,7 +157,7 @@ class RoutingConfig(ServiceConfig):
elif self._routing_provider == self.TOMTOM_PROVIDER:
self._tomtom_api_keys = self._db_config.tomtom_routing_api_keys
self._tomtom_service_params = self._db_config.tomtom_routing_service_params
self._set_monthly_quota()
self._routing_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@@ -210,9 +210,13 @@ class RoutingConfig(ServiceConfig):
def tomtom_service_params(self):
return self._tomtom_service_params
@property
def routing_quota(self):
return self._routing_quota
@property
def monthly_quota(self):
return self._monthly_quota
return self._routing_quota
@property
def period_end_date(self):
@@ -222,9 +226,6 @@ class RoutingConfig(ServiceConfig):
def soft_limit(self):
return self._soft_limit
def _set_monthly_quota(self):
self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
def _set_soft_limit(self):
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True

View File

@@ -128,7 +128,7 @@ class QuotaChecker:
return False
def __check_routing_quota(self):
user_quota = self._user_service_config.monthly_quota
user_quota = self._user_service_config.routing_quota
today = date.today()
service_type = self._user_service_config.service_type
current_used = self._user_service.used_quota(service_type, today)

View File

@@ -309,7 +309,7 @@ class TestRoutingConfig(TestCase):
self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
orgname = None
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.monthly_quota == 1000
assert config.routing_quota == 1000
def test_org_quota_overrides_user_quota(self):
self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
@@ -323,7 +323,7 @@ class TestRoutingConfig(TestCase):
self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
assert config.monthly_quota == 5000
assert config.routing_quota == 5000
def test_should_have_soft_limit_false_by_default(self):
orgname = None

View File

@@ -30,7 +30,7 @@ class TestQuotaChecker(TestCase):
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
routing_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
@@ -43,7 +43,7 @@ class TestQuotaChecker(TestCase):
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
routing_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
@@ -61,7 +61,7 @@ class TestQuotaChecker(TestCase):
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
routing_quota = 1000,
period_end_date = datetime.today(),
soft_limit = False
)
@@ -75,7 +75,7 @@ class TestQuotaChecker(TestCase):
username = self.username,
organization = None,
service_type = self.service_type,
monthly_quota = 1000,
routing_quota = 1000,
period_end_date = datetime.today(),
soft_limit = True
)