Merge remote-tracking branch 'origin/development'

314 exception safe public funcs

NEWS.md

@@ -1,3 +1,28 @@
+November 29st, 2016
+===================
+* Version 0.20.0 of the server and version 0.12.0 of the python library
+  * Added integration with the new Mapzen isochrones functionality
+
+November 25st, 2016
+===================
+* Version 0.19.0 of the server, version 0.11.0 of the python library and version 0.13.0 of the client
+  * functions to check the quota, both server and client
+  * removed the no_params from the templates (this caused trouble, not needed anymore)
+  * bug fixes: observatory quota, quotas as integers, mapzen geocoder soft limit as bool
+
+November 21st, 2016
+===================
+* Version 0.18.1 of the server and version 0.12.1 of the client
+  * Add new fields to the obs_meta_geometry due to new changes introduced in the DO 1.1.2 release
+
+November 11st, 2016
+===================
+* Version 0.18.0 of the server and version 0.12.0 of the client
+  * Added obs_legacybuildermetada functions to grab the needed metadata in the builder while making a data enrichment analysis. Closes #286
+  * Added metadata functions that will be used in the future to gather the metadata for the new data enrichment UI. Closes #287
+  * Fixed integration test for street geocoding
+  * Makefile now has a new task to create a new release in the client part, as was in the server, using make release NEW_VERSION=x.x.x
+
 November 7st, 2016
 ==================
 * Version 0.17.0 of the server and version 0.10.0 of the python package

client/.gitignore

@@ -2,9 +2,10 @@ results/
 regression.diffs
 regression.out
 20_public_functions.sql
+25_exception_safe_private_functions.sql
 30_plproxy_functions.sql
 90_grant_execute.sql
 cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
-cdb_geocoder_client--0.3.0.sql
+cdb_geocoder_client--0.3.0.sql

client/Makefile

@@ -57,9 +57,11 @@ all: $(DATA)
 .PHONY: release
 release: $(EXTENSION).control $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
-	mv *.sql old_versions
+	git mv *.sql old_versions
 	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 

client/cdb_dataservices_client--0.14.0--0.13.0.sql

@@ -0,0 +1,52 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.13.0'" to load this file. \quit
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC);
+

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.13.0'
+default_version = '0.14.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/renderer/templates/25_exception_safe_private_functions.erb

@@ -0,0 +1,61 @@
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>)
+RETURNS <%= return_type %> AS $$
+DECLARE
+  <% if not multi_row %>ret <%= return_type %>;<% end %>
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  <% if multi_row %>
+    BEGIN
+      RETURN QUERY
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+    END;
+  <% elsif multi_field %>
+    BEGIN
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% else %>
+    BEGIN
+      SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% end %>
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

client/renderer/templates/90_grant_execute.erb

@@ -1 +1,2 @@
 GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;
+GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser;

client/test/expected/25_exception_safe_private_functions_test.out

@@ -0,0 +1,56 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+-- Use regular user role
+SET ROLE test_regular_user;
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+WARNING:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
+PL/pgSQL function _cdb_geocode_street_point_exception_safe(text,text,text,text) line 21 at SQL statement
+ _cdb_geocode_street_point_exception_safe 
+------------------------------------------
+ 
+(1 row)
+
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+WARNING:  cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  PL/pgSQL function _cdb_isodistance_exception_safe(geometry,text,integer[],text[]) line 21 at RETURN QUERY
+ center | data_range | the_geom 
+--------+------------+----------
+(0 rows)
+
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+WARNING:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)"
+PL/pgSQL function _cdb_route_point_to_point_exception_safe(geometry,geometry,text,text[],text) line 21 at SQL statement
+ shape | length | duration 
+-------+--------+----------
+       |        |         
+(1 row)
+

client/test/sql/25_exception_safe_private_functions_test.sql

@@ -0,0 +1,41 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+
+-- Use regular user role
+SET ROLE test_regular_user;
+
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);

server/extension/cdb_dataservices_server--0.19.0--0.20.0.sql

@@ -0,0 +1,187 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.20.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_isolines(TEXT, TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
+    mapzen_isolines = MapzenIsolines(client, logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isolines')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  from cartodb_services.mapzen.types import coordinates_to_polygon
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  # -- Check the quota
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
+                                         logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    resp = mapzen_isochrones.isochrone(origin, mode, data_range)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      quota_service.increment_success_service_use()
+      quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isochrones')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  with metrics('cb_isodistance', user_isolines_config, logger):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.20.0--0.19.0.sql

@@ -0,0 +1,121 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.19.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_isodistance(TEXT, TEXT,  geometry(Geometry, 4326), TEXT, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_isochrones(TEXT, TEXT,  geometry(Geometry, 4326), TEXT, integer[], text[]);
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+   username TEXT,
+   orgname TEXT,
+   isotype TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
+    mapzen_isolines = MapzenIsolines(client, logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isolines')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  type = 'isochrone'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  type = 'isodistance'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  if user_isolines_config.heremaps_provider:
+    here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+    return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+    mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+    return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+  else:
+    raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.19.0'
+default_version = '0.20.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/80_isolines_helper.sql

@@ -57,11 +57,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isodistance(
    username TEXT,
    orgname TEXT,
-   isotype TEXT,
    source geometry(Geometry, 4326),
    mode TEXT,
    data_range integer[],
@@ -78,7 +76,6 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
-  # -- Check the quota
   quota_service = QuotaService(user_isolines_routing_config, redis_conn)
   if not quota_service.check_user_quota():
     raise Exception('You have reached the limit of your quota')
@@ -96,14 +93,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
 
     # -- TODO Support options properly
     isolines = {}
-    if isotype == 'isodistance':
-      for r in data_range:
-          isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
-          isolines[r] = isoline
-    elif isotype == 'isochrone':
-      for r in data_range:
-          isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
-          isolines[r] = isoline
+    for r in data_range:
+        isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+        isolines[r] = isoline
 
     result = []
     for r in data_range:
@@ -130,3 +122,67 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   finally:
     quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient, MapzenIsochrones
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  from cartodb_services.mapzen.types import coordinates_to_polygon
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  # -- Check the quota
+  quota_service = QuotaService(user_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  try:
+    mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key,
+                                         logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    resp = mapzen_isochrones.isochrone(origin, mode, data_range)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      quota_service.increment_success_service_use()
+      quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    quota_service.increment_failed_service_use()
+    logger.error('Error trying to get mapzen isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get mapzen isochrones')
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/85_isodistance.sql

@@ -1,21 +1,28 @@
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
 RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
 
   if user_isolines_config.google_services_user:
     raise Exception('This service is not available for google service users.')
 
-  if user_isolines_config.heremaps_provider:
-    here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-    return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
-  elif user_isolines_config.mapzen_provider:
-    mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-    return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
-  else:
-    raise Exception('Requested isolines provider is not available')
+  with metrics('cb_isodistance', user_isolines_config, logger):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
 $$ LANGUAGE plpythonu;
 
 -- heremaps isodistance
@@ -40,10 +47,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
-  type = 'isodistance'
 
-  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
 
   return result
 $$ LANGUAGE plpythonu;

server/extension/sql/90_isochrone.sql

@@ -47,9 +47,8 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
   plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
-  type = 'isochrone'
 
-  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
-  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
   return result
 $$ LANGUAGE plpythonu;

server/lib/python/cartodb_services/README.md

@@ -46,5 +46,7 @@ cd $(git rev-parse --show-toplevel)/test
 python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY
 ```
 
-## TODO
-- Move dependencies expressed in `requirements.txt` to `setup.py`
+## Versioning
+Once you're satisfied with your changes, it is time to bump the version number in the `setup.py`. A couple of rules:
+- **Backwards compatibility**: in general all changes shall be backwards compatible. Do not remove any code used from the server public `pl/python` functions or you'll run into problems when deploying.
+- **Semantic versioning**: we try to stick to [Semantic Versioning 2.0.0](http://semver.org/spec/v2.0.0.html)

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -4,6 +4,7 @@
 import json
 import requests
 
+from requests.adapters import HTTPAdapter
 from exceptions import *
 from cartodb_services.metrics import Traceable
 
@@ -17,6 +18,7 @@ class HereMapsGeocoder(Traceable):
     DEFAULT_GEN = 9
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ADDRESS_PARAMS = [
         'city',
@@ -88,7 +90,10 @@ class HereMapsGeocoder(Traceable):
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self.host, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 
 from exceptions import WrongParams
+from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 
 
@@ -13,6 +14,7 @@ class HereMapsRoutingIsoline(Traceable):
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -53,6 +55,9 @@ class HereMapsRoutingIsoline(Traceable):
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
         response = requests.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -2,3 +2,4 @@ from routing import MapzenRouting, MapzenRoutingResponse
 from isolines import MapzenIsolines
 from geocoder import MapzenGeocoder
 from matrix_client import MatrixClient
+from isochrones import MapzenIsochrones

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenGeocoder(Traceable):
     BASE_URL = 'https://search.mapzen.com/v1/search'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -27,7 +29,10 @@ class MapzenGeocoder(Traceable):
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
             self.add_response_data(response, self._logger)
             if response.status_code == requests.codes.ok:
@@ -49,7 +54,7 @@ class MapzenGeocoder(Traceable):
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down
-            self._logger.error('Timeout connecting to Mapzen geocoding server')
+            self._logger.error('Timeout connecting to Mapzen geocoding server', te)
             raise ServiceException('Error trying to geocode {0} using mapzen'.format(searchtext),
                                     None)
         except requests.ConnectionError as e:

server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py

@@ -0,0 +1,135 @@
+import requests
+import json
+import re
+
+from requests.adapters import HTTPAdapter
+from exceptions import WrongParams, MalformedResult, ServiceException
+from qps import qps_retry
+
+
+class MapzenIsochrones:
+    'A Mapzen Isochrones wrapper for python'
+
+    BASE_URL = 'https://matrix.mapzen.com/isochrone'
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    ACCEPTED_MODES = {
+        "walk": "pedestrian",
+        "car": "auto"
+    }
+
+    def __init__(self, app_key, logger, base_url=BASE_URL):
+        self._app_key = app_key
+        self._url = base_url
+        self._logger = logger
+
+    @qps_retry(qps=7)
+    def isochrone(self, locations, costing, ranges):
+        request_params = self._parse_request_params(locations, costing,
+                                                    ranges)
+        try:
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
+                                    timeout=(self.CONNECT_TIMEOUT,
+                                            self.READ_TIMEOUT))
+
+            if response.status_code is requests.codes.ok:
+                return self._parse_response(response)
+            elif response.status_code == requests.codes.bad_request:
+                return []
+            else:
+                self._logger.error('Error trying to get isochrones from mapzen',
+                                data={"response_status": response.status_code,
+                                      "response_reason": response.reason,
+                                      "response_content": response.text,
+                                      "reponse_url": response.url,
+                                      "response_headers": response.headers,
+                                      "locations": locations,
+                                      "costing": costing})
+                raise ServiceException('Error trying to get isochrones from mapzen',
+                                       response)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            self._logger.error('Timeout connecting to Mapzen isochrones server', exception=te)
+            raise ServiceException('Error trying to calculate isochrones using mapzen',
+                                    None)
+        except requests.ConnectionError as e:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to Mapzen isochrones server',
+                               exception=e)
+            return []
+
+    def _parse_request_params(self, locations, costing, ranges):
+        if costing in self.ACCEPTED_MODES:
+            mode_source = self.ACCEPTED_MODES[costing]
+        else:
+            raise WrongParams("{0} is not an accepted mode".format(costing))
+
+        contours = []
+        for r in ranges:
+            # range is in seconds but mapzen uses minutes
+            range_minutes = r / 60
+            contours.append({"time": range_minutes, "color": 'tbd'})
+        request_params = {
+            'json': json.dumps({'locations': [locations],
+                                'costing': mode_source,
+                                'contours': contours,
+                                'generalize': 50,
+                                'denoise': .3}),
+            'api_key': self._app_key
+        }
+
+        return request_params
+
+    def _parse_response(self, response):
+        try:
+            json_response = response.json()
+            isochrones = []
+            for feature in json_response['features']:
+                # Coordinates could have more than one isochrone. For the
+                # moment we're getting the first polygon only
+                coordinates = feature['geometry']['coordinates']
+                duration = feature['properties']['contour']
+                mapzen_response = MapzenIsochronesResponse(coordinates,
+                                                           duration)
+                isochrones.append(mapzen_response)
+            return isochrones
+        except IndexError:
+            return []
+        except KeyError:
+            self._logger.error('Non existing key for mapzen isochrones response',
+                               data={"response_status": response.status_code,
+                                     "response_reason": response.reason,
+                                     "response_content": response.text,
+                                     "reponse_url": response.url,
+                                     "response_headers": response.headers})
+            raise MalformedResult()
+        except ValueError:
+            # JSON decode error
+            self._logger.error('JSON decode error for Mapzen isochrones',
+                               data={"response_status": response.status_code,
+                                     "response_reason": response.reason,
+                                     "response_content": response.text,
+                                     "reponse_url": response.url,
+                                     "response_headers": response.headers})
+            return []
+
+
+class MapzenIsochronesResponse:
+
+    def __init__(self, coordinates, duration):
+        self._coordinates = coordinates
+        self._duration = duration
+
+    @property
+    def coordinates(self):
+        return self._coordinates
+
+    @property
+    def duration(self):
+        return self._duration

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -55,7 +55,7 @@ class QPSService:
 
     def retry(self, first_request_time, retry_count):
         elapsed = datetime.now() - first_request_time
-        if elapsed.microseconds  > (self._retry_timeout * 1000.0):
+        if elapsed.microseconds  > (self._retry_timeout * 1000.0 * 1000.0):
             raise TimeoutException()
 
         # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenRouting(Traceable):
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -46,7 +48,10 @@ class MapzenRouting(Traceable):
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/mapzen/types.py

@@ -19,6 +19,24 @@ def polyline_to_linestring(polyline):
 
     return geometry
 
+
+def coordinates_to_polygon(coordinates):
+    """Convert a Mapzen coordinates to a PostGIS polygon"""
+    result_coordinates = []
+    for coordinate in coordinates:
+        result_coordinates.append("%s %s" % (coordinate[0], coordinate[1]))
+    wkt_coordinates = ','.join(result_coordinates)
+
+    try:
+        sql = "SELECT ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326)) as geom".format(wkt_coordinates)
+        geometry = plpy.execute(sql, 1)[0]['geom']
+    except BaseException as e:
+        plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e))
+        geometry = None
+
+    return geometry
+
+
 def country_to_iso3(country):
     """ Convert country to its iso3 code """
     try:

server/lib/python/cartodb_services/cartodb_services/tools/log.py

@@ -14,7 +14,6 @@ try:
 except ImportError:
     pass
 
-
 class Logger:
 
     LEVELS = {'debug': 1, 'info': 2, 'warning': 3, 'error': 4}
@@ -66,7 +65,7 @@ class Logger:
         if self._rollbar_activated():
             try:
                 if exception:
-                    rollbar.report_exc_info(exception, extra_data=data,
+                    rollbar.report_exc_info(sys.exc_info(), extra_data=data,
                                             level=level)
                 else:
                     rollbar.report_message(text, level, extra_data=data)
@@ -102,7 +101,7 @@ class Logger:
     def _parse_log_extra_data(self, exception, data):
         extra_data = {}
         if exception:
-            type_, value_, traceback_ = exception
+            type_, value_, traceback_ = sys.exc_info()
             exception_traceback = traceback.format_tb(traceback_)
             extra_data = {"exception_type": type_, "exception_message": value_,
                           "exception_traceback": exception_traceback,

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.11.0',
+    version='0.12.3',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_heremapsrouting.py

@@ -128,8 +128,8 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
     MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
 
     def setUp(self):
-        logger = Mock()
-        self.routing = HereMapsRoutingIsoline(None, None, logger)
+        self.logger = Mock()
+        self.routing = HereMapsRoutingIsoline(None, None, self.logger)
         self.isoline_url = "{0}{1}".format(HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL,
                                      HereMapsRoutingIsoline.ISOLINE_PATH)
 

server/lib/python/cartodb_services/test/test_mapzenisochrones.py

@@ -0,0 +1,54 @@
+import mock
+import unittest
+import requests_mock
+from mock import Mock
+
+from cartodb_services.mapzen import MapzenIsochrones
+from cartodb_services.mapzen.exceptions import ServiceException
+
+requests_mock.Mocker.TEST_PREFIX = 'test_'
+
+@requests_mock.Mocker()
+class MapzenIsochronesTestCase(unittest.TestCase):
+    MAPZEN_ISOCHRONES_URL = 'https://matrix.mapzen.com/isochrone'
+
+    ERROR_RESPONSE = """{
+      "error_code": 171,
+      "error": "No suitable edges near location",
+      "status_code": 400,
+      "status": "Bad Request"
+    }"""
+
+    GOOD_RESPONSE = """{"features":[{"properties":{"opacity":0.33,"contour":15,"color":"tbd"},"type":"Feature","geometry":{"coordinates":[[-3.702579,40.430893],[-3.702193,40.430122],[-3.702579,40.430893]],"type":"LineString"}},{"properties":{"opacity":0.33,"contour":5,"color":"tbd"},"type":"Feature","geometry":{"coordinates":[[-3.703050,40.424995],[-3.702546,40.424694],[-3.703050,40.424995]],"type":"LineString"}}],"type":"FeatureCollection"}"""
+
+    def setUp(self):
+        self.logger = Mock()
+        self.mapzen_isochrones = MapzenIsochrones('matrix-xxxxx', self.logger)
+
+    def test_calculate_isochrone(self, req_mock):
+        req_mock.register_uri('GET', self.MAPZEN_ISOCHRONES_URL,
+                               text=self.GOOD_RESPONSE)
+
+        response = self.mapzen_isochrones.isochrone([-41.484375, 28.993727],
+                                                    'walk', [300, 900])
+
+        self.assertEqual(len(response), 2)
+        self.assertEqual(response[0].coordinates, [[-3.702579,40.430893],[-3.702193,40.430122],[-3.702579,40.430893]])
+        self.assertEqual(response[0].duration, 15)
+        self.assertEqual(response[1].coordinates, [[-3.703050,40.424995],[-3.702546,40.424694],[-3.703050,40.424995]])
+        self.assertEqual(response[1].duration, 5)
+
+    def test_calculate_isochrone_error_400_returns_empty(self, req_mock):
+        req_mock.register_uri('GET', self.MAPZEN_ISOCHRONES_URL,
+                               text=self.ERROR_RESPONSE, status_code=400)
+
+        response = self.mapzen_isochrones.isochrone([-41.484375, 28.993727],
+                                                    'walk', [300, 900])
+        self.assertEqual(response, [])
+
+    def test_calculate_isochrone_error_500_returns_exception(self, req_mock):
+        req_mock.register_uri('GET', self.MAPZEN_ISOCHRONES_URL,
+                              text=self.ERROR_RESPONSE, status_code=500)
+        with self.assertRaises(ServiceException):
+            self.mapzen_isochrones.isochrone([-41.484375, 28.993727],
+                                             'walk', [300, 900])