Merge pull request #517 from CartoDB/development

Release 0.33 server and 0.20 for python library

.gitignore

@@ -5,3 +5,5 @@ cartodb_services.egg-info/
 build/
 dist/
 .vscode/
+.idea/
+venv/

NEWS.md

@@ -1,4 +1,59 @@
 
+Aug 27th, 2018
+==============
+* Version `0.33.0` of the server, and `0.20.0` of the Python library.
+    * Remove the obs_snapshot quota and now the snapshot functions uses obs_general quota
+
+Jul 19th, 2018
+==============
+* Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.1` of the Python library.
+    * Support for batch street-level geocoding.
+
+May 7th, 2018
+=============
+* Version `0.24.0` of the client, `0.31.0` of the server, and `0.18.0` of the python library.
+    * Support for TomTom routing, geocoding and isolines.
+
+March 28th, 2018
+================
+* Version `0.30.5` of server side and `0.17.6` of the python library
+    * All the returned polygons are now always MULTIPOLYGON #488
+
+March 27th, 2018
+================
+* Version `0.17.5` of python library
+    * Avoid reaching provider for empty geocodings (but still incrementing empty service use) #476
+
+March 16th, 2018
+================
+* Version `0.30.3` of server side
+    * Fix problem with invalid Mapbox isolines #483
+* Version `0.30.4` of server side
+    * Added ST_CollectionExtract to ST_MakeValid for Mapbox isolines to avoid non-polygonal geometries #486
+
+March 14th, 2018
+================
+* Version `0.17.4` of the python library
+    * Fix bug with previous version when checking quotas #480
+* Version `0.17.3` of the python library
+    * Fix bug with Mapbox routing not using the proper quota value #477
+
+February 22th, 2018
+==================
+* Version `0.17.2` of the python library
+    * Fix bug with Mapbox isolines not stopping at the seacoast #471
+
+February 27th, 2018
+==================
+* Version `0.17.1` of the python library
+    * Fix bug when the mapzen credentials are not in the db config and we keep getting them
+
+February 22th, 2018
+==================
+* Version `0.17.0` of the python library
+    * Change default provider to Mapbox
+    * Remove the obligatory nature of the Mapzen configuration due to its deprecation as provider
+
 February 13th, 2018
 ==================
 * Version `0.16.7` of the python library

README.md

@@ -171,6 +171,15 @@ SELECT CDB_Conf_SetConf(
 );
 ```
 
+#### TomTom configuration
+
+```sql
+SELECT CDB_Conf_SetConf(
+    'tomtom_conf',
+    '{"routing": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}, "isolines": {"api_keys": ["your_api_key"], "monthly_quota": 1500000}}'
+);
+```
+
 #### Data Observatory
 
 ```sql

client/carto-package.json

@@ -0,0 +1,13 @@
+{
+  "name": "dataservices-api-client-extension",
+  "current_version": {
+    "requires": {
+      "postgresql": "^10.0.0",
+      "postgis": "^2.4.0.0",
+      "carto_postgresql_ext": "^0.23.0"
+    },
+    "works_with": {
+      "dataservices-api-server-extension": "^0.32.0"
+    }
+  }
+}

client/cdb_dataservices_client--0.24.0--0.25.0.sql

@@ -0,0 +1,276 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
+
+-- Taken from https://stackoverflow.com/a/48013356/351721
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
+    SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
+$f$ LANGUAGE sql IMMUTABLE;--
+
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
+CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
+    service cdb_dataservices_client.service_type,
+    monthly_quota NUMERIC,
+    used_quota NUMERIC,
+    soft_limit BOOLEAN,
+    provider TEXT,
+    max_batch_size NUMERIC
+);
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF service_quota_info_batch AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+BEGIN
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;
+
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ()
+RETURNS SETOF service_quota_info_batch AS $$
+DECLARE
+
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text)
+RETURNS SETOF service_quota_info_batch AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) TO publicuser;

client/cdb_dataservices_client--0.25.0--0.24.0.sql

@@ -0,0 +1,29 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_count_estimate(query text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_quota_info_batch();
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, text, text, text, text, integer);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ();
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (text, text);
+
+DROP TYPE IF EXISTS cdb_dataservices_client.service_quota_info_batch;
+
+DROP TYPE IF EXISTS cdb_dataservices_client.geocoding;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.23.0'
+default_version = '0.25.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.23.0--0.24.0.sql

@@ -0,0 +1,202 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.24.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS Geometry AS $$
+DECLARE
+  ret Geometry;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS Geometry AS $$
+DECLARE
+  ret Geometry;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    SELECT cdb_dataservices_client._cdb_tomtom_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+         RETURN ret;
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isochrone(username, orgname, source, mode, range, options);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (source geometry(Geometry, 4326) ,mode text ,range integer[] ,options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_tomtom_isodistance(username, orgname, source, mode, range, options);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_tomtom_geocode_street_point (username, orgname, searchtext, city, state_province, country);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone (username, orgname, source, mode, range, options);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance (username, orgname, source, mode, range, options);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_tomtom_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[] )  TO publicuser;

client/old_versions/cdb_dataservices_client--0.24.0--0.23.0.sql

@@ -0,0 +1,25 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.23.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_geocode_street_point (text, text, text, text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_isochrone (geometry(Geometry, 4326), text, integer[], text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_tomtom_isodistance (geometry(Geometry, 4326), text, integer[], text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point_exception_safe (text, text, text, text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone_exception_safe (geometry(Geometry, 4326), text, integer[], text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance_exception_safe (geometry(Geometry, 4326), text, integer[], text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isochrone (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_tomtom_isodistance (username text, orgname text, source geometry(Geometry, 4326), mode text, range integer[], options text[]);

client/renderer/interface.yaml

@@ -70,6 +70,13 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: _cdb_bulk_geocode_street_point
+  return_type: SETOF cdb_dataservices_client.geocoding
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: searches,  type: jsonb } # Array of JSON objects with id, address, city, state and country fields
+
 - name: cdb_here_geocode_street_point
   return_type: Geometry
   params:
@@ -94,6 +101,14 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: cdb_tomtom_geocode_street_point
+  return_type: Geometry
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
 - name: cdb_mapzen_geocode_street_point
   return_type: Geometry
   params:
@@ -132,6 +147,16 @@
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
 
+- name: cdb_tomtom_isochrone
+  return_type: SETOF cdb_dataservices_client.isoline
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: mode, type: text }
+    - { name: range, type: "integer[]" }
+    - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
+
 - name: cdb_mapzen_isochrone
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
@@ -152,6 +177,16 @@
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
 
+- name: cdb_tomtom_isodistance
+  return_type: SETOF cdb_dataservices_client.isoline
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: mode, type: text }
+    - { name: range, type: "integer[]" }
+    - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
+
 - name: cdb_mapzen_isodistance
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
@@ -482,6 +517,13 @@
   params:
     - {}
 
+- name: cdb_service_quota_info_batch
+  return_type: SETOF service_quota_info_batch
+  multi_row: true
+  multi_field: true
+  params:
+    - {}
+
 - name: cdb_enough_quota
   return_type: BOOLEAN
   params:

client/sql/05_utils.sql

@@ -0,0 +1,20 @@
+-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
+CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
+
+-- Taken from https://stackoverflow.com/a/48013356/351721
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
+    SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
+$f$ LANGUAGE sql IMMUTABLE;

client/sql/16_custom_types.sql

@@ -4,6 +4,12 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
     the_geom geometry(Multipolygon,4326)
 );
 
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
 CREATE TYPE cdb_dataservices_client.simple_route AS (
     shape geometry(LineString,4326),
     length real,
@@ -35,3 +41,12 @@ CREATE TYPE cdb_dataservices_client.service_quota_info AS (
     soft_limit BOOLEAN,
     provider TEXT
 );
+
+CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
+    service cdb_dataservices_client.service_type,
+    monthly_quota NUMERIC,
+    used_quota NUMERIC,
+    soft_limit BOOLEAN,
+    provider TEXT,
+    max_batch_size NUMERIC
+);

client/sql/21_bulk_geocoding_functions.sql

@@ -0,0 +1,76 @@
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+BEGIN
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;

client/sql/95_grant_execute_manual.sql

@@ -1,3 +1,7 @@
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser;
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser;
 GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser;

client/test/expected/00_installation_test.out

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

client/test/expected/21_bulk_geocoding_functions_test.out

@@ -0,0 +1,21 @@
+\set VERBOSITY terse
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
+  SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
+$$ LANGUAGE SQL;
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT FALSE;
+$$ LANGUAGE SQL;
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+ERROR:  Remaining quota: 0. Estimated cost: 1
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+ERROR:  Remaining quota: 0. Estimated cost: 1
+DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
+DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;

client/test/sql/00_installation_test.sql

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;

client/test/sql/21_bulk_geocoding_functions_test.sql

@@ -0,0 +1,26 @@
+\set VERBOSITY terse
+
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
+  SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
+$$ LANGUAGE SQL;
+
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT FALSE;
+$$ LANGUAGE SQL;
+
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+
+DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
+DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
+
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;
+

server/extension/Makefile

@@ -83,3 +83,6 @@ deploy: release_remove_parallel_deploy
 	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
 install: deploy
+
+reinstall: install
+	psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"

server/extension/carto-package.json

@@ -0,0 +1,14 @@
+{
+  "name": "dataservices-api-server-extension",
+  "current_version": {
+    "requires": {
+      "postgresql": "^10.0.0",
+      "postgis": "^2.4.0.0",
+      "carto_postgresql_ext": "^0.23.0"
+    },
+    "works_with": {
+      "dataservices-api-server-python-lib": "^0.19.1",
+      "observatory-server-extension": "^1.9.0"
+    }
+  }
+}

server/extension/cdb_dataservices_server--0.32.0--0.33.0.sql

@@ -0,0 +1,204 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.33.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  return user_obs_config.connection_str
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_segment_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetSegmentSnapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.33.0--0.32.0.sql

@@ -0,0 +1,216 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.32.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_obs_snapshot_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import ObservatorySnapshotConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = obs_snapshot_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
+  username TEXT,
+  orgname TEXT)
+RETURNS text AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  return user_obs_config.connection_str
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  time_span TEXT DEFAULT NULL,
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_demographic_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS json AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          quota_service.increment_success_service_use()
+          return result[0]['snapshot']
+        else:
+          quota_service.increment_empty_service_use()
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to obs_get_segment_snapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF json AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetSegmentSnapshot(geom, geometry_level);
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
+  username TEXT,
+  orgname TEXT,
+  geom geometry(Geometry, 4326),
+  geometry_level TEXT DEFAULT NULL)
+RETURNS SETOF JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
+        if result:
+          resp = []
+          for element in result:
+            value = element['snapshot']
+            resp.append(value)
+          quota_service.increment_success_service_use()
+          return resp
+        else:
+          quota_service.increment_empty_service_use()
+          return []
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use()
+        logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetSegmentSnapshot')
+    finally:
+        quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.30.2'
+default_version = '0.33.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.30.2--0.30.3.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.2.sql

@@ -0,0 +1,70 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.3--0.30.4.sql

@@ -0,0 +1,69 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.3.sql

@@ -0,0 +1,69 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.3'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.4--0.30.5.sql

@@ -0,0 +1,68 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.5'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.5--0.30.4.sql

@@ -0,0 +1,68 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.4'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
+  from cartodb_services.mapbox.types import TRANSPORT_MODE_TO_MAPBOX
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.mapbox_isolines_config import MapboxIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', MapboxIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = MapboxMatrixClient(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+    mapbox_isolines = MapboxIsolines(client, service_manager.logger)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_MAPBOX.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get Mapbox isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get Mapbox isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.30.5--0.31.0.sql

@@ -0,0 +1,470 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.31.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT)
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
+  username TEXT,
+  orgname TEXT,
+  origin geometry(Point, 4326),
+  destination geometry(Point, 4326),
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_point', user_routing_config, logger, params):
+    waypoints = [origin, destination]
+
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.tomtom import TomTomGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using TomTom')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = tomtom_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          service_manager.quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          service_manager.quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+-- tomtom isodistance
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  params = {'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+-- tomtom isochrone
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+  return result
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.31.0--0.30.5.sql

@@ -0,0 +1,182 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.30.5'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_route_with_waypoints(TEXT, TEXT, geometry(Point, 4326)[], TEXT);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
+  username TEXT,
+  orgname TEXT,
+  origin geometry(Point, 4326),
+  destination geometry(Point, 4326),
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  with metrics('cdb_route_with_point', user_routing_config, logger):
+    waypoints = [origin, destination]
+
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE ;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT,
+  options text[] DEFAULT ARRAY[]::text[],
+  units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger):
+    if user_routing_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    else:
+      raise Exception('Requested routing method is not available')
+$$ LANGUAGE plpythonu STABLE ;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE ;
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_tomtom_isochrones(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  with metrics('cb_isodistance', user_isolines_config, logger):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu STABLE ;
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  if user_isolines_config.google_services_user:
+    raise Exception('This service is not available for google service users.')
+
+  with metrics('cb_isochrone', user_isolines_config, logger):
+    if user_isolines_config.heremaps_provider:
+      here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapzen_provider:
+      mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.mapbox_provider:
+      mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    else:
+      raise Exception('Requested isolines provider is not available')
+$$ LANGUAGE plpythonu STABLE ;
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_tomtom_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);

server/extension/old_versions/cdb_dataservices_server--0.31.0--0.32.0.sql

@@ -0,0 +1,148 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
+                 WHERE pg_type.typname = 'service_quota_info_batch'
+                 AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
+    CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS (
+      service cdb_dataservices_server.service_type,
+      monthly_quota NUMERIC,
+      used_quota NUMERIC,
+      soft_limit BOOLEAN,
+      provider TEXT,
+      max_batch_size NUMERIC
+    );
+  END IF;
+END $$;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
+  username TEXT,
+  orgname TEXT)
+RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
+  from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+
+  ret = []
+  for info in sqi:
+      if info['service'] == 'hires_geocoder':
+          provider = info['provider']
+          batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
+          if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
+              max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
+          else:
+              max_batch_size = 1
+
+          info['max_batch_size'] = max_batch_size
+      else:
+          info['max_batch_size'] = 1
+
+      ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
+
+  return ret
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
+CREATE TYPE cdb_dataservices_server.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'searches': searches}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.google import GoogleMapsBulkGeocoder
+
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.here import HereMapsBulkGeocoder
+
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+  from cartodb_services.tomtom import TomTomBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder
+  from cartodb_services.mapbox import MapboxBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.32.0--0.31.0.sql

@@ -0,0 +1,15 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_quota_info_batch(TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocode_street_point(TEXT, TEXT, jsonb);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_google_geocode_street_point(TEXT, TEXT, jsonb);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(TEXT, TEXT, jsonb);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(TEXT, TEXT, jsonb);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(TEXT, TEXT, jsonb);
+DROP TYPE IF EXISTS cdb_dataservices_server.geocoding;
+DROP TYPE IF EXISTS cdb_dataservices_server.service_quota_info_batch;

server/extension/sql/100_routing_helper.sql

@@ -65,6 +65,67 @@ RETURNS cdb_dataservices_server.simple_route AS $$
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_route_with_waypoints(
+  username TEXT,
+  orgname TEXT,
+  waypoints geometry(Point, 4326)[],
+  mode TEXT)
+RETURNS cdb_dataservices_server.simple_route AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomRouting
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.polyline import polyline_to_linestring
+  from cartodb_services.refactor.service.tomtom_routing_config import TomTomRoutingConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('routing', TomTomRoutingConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    client = TomTomRouting(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if not waypoints or len(waypoints) < 2:
+      service_manager.logger.info("Empty origin or destination")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    if len(waypoints) > 25:
+      service_manager.logger.info("Too many waypoints (max 25)")
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+
+    waypoint_coords = []
+    for waypoint in waypoints:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % waypoint)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
+      waypoint_coords.append(Coordinate(lon,lat))
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = client.directions(waypoint_coords, profile)
+    if resp and resp.shape:
+      shape_linestring = polyline_to_linestring(resp.shape)
+      if shape_linestring:
+        service_manager.quota_service.increment_success_service_use()
+        return [shape_linestring, resp.length, int(round(resp.duration))]
+      else:
+        service_manager.quota_service.increment_empty_service_use()
+        return [None, None, None]
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return [None, None, None]
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to calculate TomTom routing', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to calculate TomTom routing')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_with_waypoints(
   username TEXT,
   orgname TEXT,

server/extension/sql/105_route_between_points.sql

@@ -18,7 +18,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  with metrics('cdb_route_with_point', user_routing_config, logger):
+  params = {'origin': origin, 'destination': destination, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_point', user_routing_config, logger, params):
     waypoints = [origin, destination]
 
     if user_routing_config.mapzen_provider:
@@ -29,6 +31,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
       result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -53,7 +59,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  with metrics('cdb_route_with_waypoints', user_routing_config, logger):
+  params = {'waypoints': waypoints, 'mode': mode, 'options': options, 'units': units}
+
+  with metrics('cdb_route_with_waypoints', user_routing_config, logger, params):
     if user_routing_config.mapzen_provider:
       mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
       result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode])
@@ -62,6 +70,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$
       mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapbox_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
       result = plpy.execute(mapbox_plan, [username, orgname, waypoints, mode])
       return [result[0]['shape'],result[0]['length'], result[0]['duration']]
+    elif user_routing_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_route_with_waypoints($1, $2, $3, $4) as route;", ["text", "text", "geometry(Point, 4326)[]", "text"])
+      result = plpy.execute(tomtom_plan, [username, orgname, waypoints, mode])
+      return [result[0]['shape'],result[0]['length'], result[0]['duration']]
     else:
       raise Exception('Requested routing method is not available')
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/110_data_observatory_augmentation.sql

@@ -10,8 +10,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._obs_server_conn_str(
 RETURNS text AS $$
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   return user_obs_config.connection_str
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -41,8 +41,8 @@ RETURNS json AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -94,8 +94,8 @@ RETURNS SETOF JSON AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -150,8 +150,8 @@ RETURNS json AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]
@@ -201,8 +201,8 @@ RETURNS SETOF JSON AS $$
 
   plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
   redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
-  user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
   logger_config = GD["logger_config"]

server/extension/sql/15_config_helper.sql

@@ -90,20 +90,6 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
-RETURNS boolean AS $$
-  cache_key = "user_obs_snapshot_config_{0}".format(username)
-  if cache_key in GD:
-    return False
-  else:
-    from cartodb_services.metrics import ObservatorySnapshotConfig
-    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
-    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
-    obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
-    GD[cache_key] = obs_snapshot_config
-    return True
-$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
-
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_obs_config_{0}".format(username)

server/extension/sql/200_quotas.sql

@@ -27,6 +27,23 @@ BEGIN
   END IF;
 END $$;
 
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type inner join pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
+                 WHERE pg_type.typname = 'service_quota_info_batch'
+                 AND pg_namespace.nspname = 'cdb_dataservices_server') THEN
+    CREATE TYPE cdb_dataservices_server.service_quota_info_batch AS (
+      service cdb_dataservices_server.service_type,
+      monthly_quota NUMERIC,
+      used_quota NUMERIC,
+      soft_limit BOOLEAN,
+      provider TEXT,
+      max_batch_size NUMERIC
+    );
+  END IF;
+END $$;
+
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
   username TEXT,
   orgname TEXT)
@@ -92,6 +109,35 @@ RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info_batch(
+  username TEXT,
+  orgname TEXT)
+RETURNS SETOF cdb_dataservices_server.service_quota_info_batch AS $$
+  from cartodb_services.bulk_geocoders import BATCH_GEOCODER_CLASS_BY_PROVIDER
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  sqi = plpy.execute("SELECT * from cdb_dataservices_server.cdb_service_quota_info({0},{1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+
+  ret = []
+  for info in sqi:
+      if info['service'] == 'hires_geocoder':
+          provider = info['provider']
+          batch_geocoder_class = BATCH_GEOCODER_CLASS_BY_PROVIDER.get(provider, None)
+          if batch_geocoder_class and hasattr(batch_geocoder_class, 'MAX_BATCH_SIZE'):
+              max_batch_size = batch_geocoder_class.MAX_BATCH_SIZE
+          else:
+              max_batch_size = 1
+
+          info['max_batch_size'] = max_batch_size
+      else:
+          info['max_batch_size'] = 1
+
+      ret += [[info['service'], info['monthly_quota'], info['used_quota'], info['soft_limit'], info['provider'], info['max_batch_size']]]
+
+  return ret
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
   username TEXT,
   orgname TEXT,

server/extension/sql/20_geocode_street.sql

@@ -12,7 +12,9 @@ RETURNS Geometry AS $$
   logger_config = GD["logger_config"]
   logger = Logger(logger_config)
 
-  with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
+  params = {'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
     if user_geocoder_config.heremaps_geocoder:
       here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
       return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
@@ -25,6 +27,9 @@ RETURNS Geometry AS $$
     elif user_geocoder_config.mapbox_geocoder:
       mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
       return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
     else:
       raise Exception('Requested geocoder is not available')
 
@@ -82,8 +87,21 @@ RETURNS Geometry AS $$
   plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
   user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
 
-  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
-  return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+  mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
 
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
@@ -233,8 +251,54 @@ RETURNS Geometry AS $$
   except BaseException as e:
     import sys
     service_manager.quota_service.increment_failed_service_use()
-    service_manager.logger.error('Error trying to geocode street point using mapbox', sys.exc_info(), data={"username": username, "orgname": orgname})
-    raise Exception('Error trying to geocode street point using mapbox')
+    service_manager.logger.error('Error trying to geocode street point using Mapbox', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using Mapbox')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.tomtom import TomTomGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = TomTomGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using TomTom', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using TomTom')
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/21_bulk_geocode_street.sql

@@ -0,0 +1,97 @@
+-- TODO: could cartodb_id be replaced by rowid, maybe needing extra care for offset?
+CREATE TYPE cdb_dataservices_server.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'searches': searches}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_google_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.google import GoogleMapsBulkGeocoder
+
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  geocoder = GoogleMapsBulkGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_heremaps_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.here import HereMapsBulkGeocoder
+
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  geocoder = HereMapsBulkGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_tomtom_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.tomtom_geocoder_config import TomTomGeocoderConfigBuilder
+  from cartodb_services.tomtom import TomTomBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', TomTomGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = TomTomBulkGeocoder(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_mapbox_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.mapbox_geocoder_config import MapboxGeocoderConfigBuilder
+  from cartodb_services.mapbox import MapboxBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', MapboxGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = MapboxBulkGeocoder(service_manager.config.mapbox_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+

server/extension/sql/80_isolines_helper.sql

@@ -169,7 +169,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
         # -- TODO encapsulate this block into a func/method
         locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
         wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
-        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3)) as geom".format(wkt_coordinates)
         multipolygon = plpy.execute(sql, 1)[0]['geom']
       else:
         multipolygon = None
@@ -188,6 +188,70 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isodistance(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    # -- TODO Support options properly
+    isolines = {}
+    for r in data_range:
+        isoline = tomtom_isolines.calculate_isodistance(origin, r, profile)
+        isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l.longitude, l.latitude) for l in locations])
+        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326)),3) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    service_manager.quota_service.increment_success_service_use()
+    service_manager.quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isolines', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isolines')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isochrones(
    username TEXT,
    orgname TEXT,
@@ -311,3 +375,63 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_tomtom_isochrones(
+   username TEXT,
+   orgname TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.tomtom import TomTomIsolines
+  from cartodb_services.tomtom.types import TRANSPORT_MODE_TO_TOMTOM
+  from cartodb_services.tools import Coordinate
+  from cartodb_services.tools.coordinates import coordinates_to_polygon
+  from cartodb_services.refactor.service.tomtom_isolines_config import TomTomIsolinesConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('isolines', TomTomIsolinesConfigBuilder, username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    tomtom_isolines = TomTomIsolines(service_manager.config.tomtom_api_key, service_manager.logger, service_manager.config.service_params)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = Coordinate(lon,lat)
+    else:
+      raise Exception('source is NULL')
+
+    profile = TRANSPORT_MODE_TO_TOMTOM.get(mode)
+
+    resp = tomtom_isolines.calculate_isochrone(origin, data_range, profile)
+
+    if resp:
+      result = []
+      for isochrone in resp:
+        result_polygon = coordinates_to_polygon(isochrone.coordinates)
+        if result_polygon:
+          service_manager.quota_service.increment_success_service_use()
+          result.append([source, isochrone.duration, result_polygon])
+        else:
+          service_manager.quota_service.increment_empty_service_use()
+          result.append([source, isochrone.duration, None])
+      service_manager.quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_isolines_service_use(len(result))
+      return result
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return []
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to get TomTom isochrones', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to get TomTom isochrones')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER STABLE PARALLEL RESTRICTED;

server/extension/sql/85_isodistance.sql

@@ -14,7 +14,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   if user_isolines_config.google_services_user:
     raise Exception('This service is not available for google service users.')
 
-  with metrics('cb_isodistance', user_isolines_config, logger):
+  params = {'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isodistance', user_isolines_config, logger, params):
     if user_isolines_config.heremaps_provider:
       here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
       return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
@@ -24,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     elif user_isolines_config.mapbox_provider:
       mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
       return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -70,3 +75,17 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
 
   return result
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+-- tomtom isodistance
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/90_isochrone.sql

@@ -14,7 +14,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   if user_isolines_config.google_services_user:
     raise Exception('This service is not available for google service users.')
 
-  with metrics('cb_isochrone', user_isolines_config, logger):
+  params = {'source': source, 'mode': mode, 'range': range, 'options': options}
+
+  with metrics('cdb_isochrone', user_isolines_config, logger, params):
     if user_isolines_config.heremaps_provider:
       here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
       return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
@@ -24,6 +26,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
     elif user_isolines_config.mapbox_provider:
       mapbox_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapbox_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
       return plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
+    elif user_isolines_config.tomtom_provider:
+      tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_tomtom_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+      return plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
     else:
       raise Exception('Requested isolines provider is not available')
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
@@ -68,3 +73,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   result = plpy.execute(mapbox_plan, [username, orgname, source, mode, range, options])
   return result
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+-- tomtom isochrone
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_tomtom_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+
+  tomtom_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_tomtom_isochrones($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(tomtom_plan, [username, orgname, source, mode, range, options])
+  return result
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/test/expected/00_install_test.out

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;
@@ -37,6 +39,12 @@ SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routi
  
 (1 row)
 
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
  cdb_conf_setconf 
 ------------------

server/extension/test/expected/50_namedplaces_test.out

@@ -1,21 +1,21 @@
 -- Check that the public function is callable, even with no data
 -- It should return NULL
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
  cdb_geocode_namedplace_point 
 ------------------------------
  
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
  cdb_geocode_namedplace_point 
 ------------------------------
  
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
  cdb_geocode_namedplace_point 
 ------------------------------
  
@@ -35,42 +35,42 @@ INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain', 'Spain'], 'E
 INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
 -- This should return the point inserted above
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
 (1 row)
 
 SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
-WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder
+WARNING:  Error geocoding namedplace using geocode street point, falling back to internal geocoder. Exception: spiexceptions.ExternalRoutineException: cartodb_services.metrics.config.ConfigException: There is no user config available. Please check your configuration.'
             cdb_geocode_namedplace_point            
 ----------------------------------------------------
  0101000020E6100000637FD93D7958E63F2ECA6C9049A24340

server/extension/test/sql/00_install_test.sql

@@ -1,3 +1,5 @@
+-- Only show warning or error messages in the tests output
+SET client_min_messages TO WARNING;
 -- Install dependencies
 CREATE EXTENSION postgis;
 CREATE EXTENSION plpythonu;
@@ -14,6 +16,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 

server/lib/python/cartodb_services/README.md

@@ -24,7 +24,7 @@ NOTE: a system installation is required at present because the library is meant
 
 
 ## Running the unit tests
-Just run `nosetests test/`
+Just run `MAPBOX_API_KEY=xxx nosetests test/`
 ```shell
 $ nosetests test/
 ......................................................................................................

server/lib/python/cartodb_services/carto-package.json

@@ -0,0 +1,8 @@
+{
+  "name": "dataservices-api-server-python-lib",
+  "current_version": {
+    "requires": {
+      "python": "~2.7.0"
+    }
+  }
+}

server/lib/python/cartodb_services/cartodb_services/__init__.py

@@ -33,3 +33,5 @@ def _reset():
 
     plpy = None
     GD = None
+
+from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -0,0 +1,11 @@
+from google import GoogleMapsBulkGeocoder
+from here import HereMapsBulkGeocoder
+from tomtom import TomTomBulkGeocoder
+from mapbox import MapboxBulkGeocoder
+
+BATCH_GEOCODER_CLASS_BY_PROVIDER = {
+    'google': GoogleMapsBulkGeocoder,
+    'heremaps': HereMapsBulkGeocoder,
+    'tomtom': TomTomBulkGeocoder,
+    'mapbox': MapboxBulkGeocoder
+}

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -0,0 +1,176 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+from tools import QuotaExceededException, Logger
+from collections import namedtuple
+import json
+
+
+PRECISION_PRECISE = 'precise'
+PRECISION_INTERPOLATED = 'interpolated'
+
+def geocoder_metadata(relevance, precision, match_types):
+    return {
+        'relevance': round(relevance, 2),
+        'precision': precision,
+        'match_types': match_types
+    }
+
+
+def geocoder_error_response(message):
+    return [[], {'error': message}]
+
+
+# Single empty result
+EMPTY_RESPONSE = [[], {}]
+# HTTP 429 and related
+TOO_MANY_REQUESTS_ERROR_RESPONSE = geocoder_error_response('Rate limit exceeded')
+# Full empty _batch_geocode response
+EMPTY_BATCH_RESPONSE = []
+
+
+def compose_address(street, city=None, state=None, country=None):
+    return ', '.join(filter(None, [street, city, state, country]))
+
+
+def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches_string):
+    plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+    logger_config = GD["logger_config"]
+
+    logger = Logger(logger_config)
+
+    success_count, failed_count, empty_count = 0, 0, 0
+
+    try:
+        searches = json.loads(searches_string)
+    except Exception as e:
+        logger.error('Parsing searches', exception=e, data={'searches': searches_string})
+        raise e
+
+    try:
+        service_manager.assert_within_limits(quota=False)
+        geocode_results = geocoder.bulk_geocode(searches)
+        results = []
+        a_failed_one = None
+        if not geocode_results == EMPTY_BATCH_RESPONSE:
+            for result in geocode_results:
+                metadata = result[2] if len(result) > 2 else {}
+                try:
+                    if metadata.get('error', None):
+                        results.append([result[0], None, json.dumps(metadata)])
+                        a_failed_one = result
+                        failed_count += 1
+                    elif result[1] and len(result[1]) == 2:
+                        plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"])
+                        point = plpy.execute(plan, result[1], 1)[0]
+                        results.append([result[0], point['the_geom'], json.dumps(metadata)])
+                        success_count += 1
+                    else:
+                        results.append([result[0], None, json.dumps(metadata)])
+                        empty_count += 1
+                except Exception as e:
+                    import sys
+                    logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
+                    metadata['processing_error'] = 'Error: {}'.format(e.message)
+                    results.append([result[0], None, json.dumps(metadata)])
+                    failed_count += 1
+
+        missing_count = len(searches) - success_count - failed_count - empty_count
+
+        if a_failed_one:
+            logger.warning("failed geocoding",
+                           data={
+                               "username": username,
+                               "orgname": orgname,
+                               "failed": str(a_failed_one),
+                               "success_count": success_count,
+                               "empty_count": empty_count,
+                               "missing_count": missing_count,
+                               "failed_count": failed_count
+                           })
+        else:
+            logger.debug("finished geocoding",
+                           data={
+                               "username": username,
+                               "orgname": orgname,
+                               "success_count": success_count,
+                               "empty_count": empty_count,
+                               "missing_count": missing_count,
+                               "failed_count": failed_count
+                           })
+        service_manager.quota_service.increment_success_service_use(success_count)
+        service_manager.quota_service.increment_empty_service_use(empty_count + missing_count)
+        service_manager.quota_service.increment_failed_service_use(failed_count)
+
+        return results
+    except QuotaExceededException as qe:
+        logger.debug('QuotaExceededException at run_street_point_geocoder', qe,
+                     data={"username": username, "orgname": orgname})
+        service_manager.quota_service.increment_failed_service_use(len(searches))
+        return []
+    except BaseException as e:
+        import sys
+        service_manager.quota_service.increment_failed_service_use(len(searches))
+        service_manager.logger.error('Error trying to bulk geocode street point', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to bulk geocode street')
+    finally:
+        service_manager.quota_service.increment_total_service_use(len(searches))
+
+
+StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state country')
+
+
+class StreetPointBulkGeocoder:
+    """
+    Classes extending StreetPointBulkGeocoder should implement:
+        * _batch_geocode(street_geocoder_searches)
+        * MAX_BATCH_SIZE
+
+    If they want to provide an alternative serial (for small batches):
+        * _should_use_batch(street_geocoder_searches)
+        * _serial_geocode(street_geocoder_searches)
+    """
+
+    SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country']
+
+    def bulk_geocode(self, decoded_searches):
+        """
+        :param decoded_searches: JSON array
+        :return: array of tuples with three elements:
+            * id
+            * latitude and longitude (array of two elements)
+            * empty array (future use: metadata)
+        """
+        street_geocoder_searches = []
+        for search in decoded_searches:
+            search_id, address, city, state, country = \
+                [search.get(k, None) for k in self.SEARCH_KEYS]
+            street_geocoder_searches.append(
+                StreetGeocoderSearch(search_id, address, city, state, country))
+
+        if len(street_geocoder_searches) > self.MAX_BATCH_SIZE:
+            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
+        try:
+            if self._should_use_batch(street_geocoder_searches):
+                return self._batch_geocode(street_geocoder_searches)
+            else:
+                return self._serial_geocode(street_geocoder_searches)
+        except Exception as e:
+            msg = "Error running geocode: {}".format(e)
+            self._logger.error(msg, e)
+            errors = [geocoder_error_response(msg)] * len(decoded_searches)
+            results = []
+            for s, r in zip(decoded_searches, errors):
+                results.append((s['id'], r[0], r[1]))
+            return results
+
+    def _batch_geocode(self, street_geocoder_searches):
+        raise NotImplementedError('Subclasses must implement _batch_geocode')
+
+    def _serial_geocode(self, street_geocoder_searches):
+        raise NotImplementedError('Subclasses must implement _serial_geocode')
+
+    def _should_use_batch(self, street_geocoder_searches):
+        return True
+
+

server/lib/python/cartodb_services/cartodb_services/google/__init__.py

@@ -1 +1,2 @@
 from geocoder import GoogleMapsGeocoder
+from bulk_geocoder import GoogleMapsBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py

@@ -0,0 +1,64 @@
+from multiprocessing import Pool
+from exceptions import MalformedResult
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.geocoder import compose_address, geocoder_error_response
+from cartodb_services.google import GoogleMapsGeocoder
+
+
+def async_geocoder(geocoder, address, components):
+    return geocoder.geocode(address=address, components=components)
+
+
+class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder):
+    """A Google Maps Geocoder wrapper for python"""
+    MAX_BATCH_SIZE = 1000
+    MIN_BATCHED_SEARCH = 2  # Batched is a parallelization
+    PARALLEL_PROCESSES = 13
+
+    def __init__(self, client_id, client_secret, logger):
+        GoogleMapsGeocoder.__init__(self, client_id, client_secret, logger)
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            (cartodb_id, street, city, state, country) = search
+            try:
+                lng_lat, metadata = self.geocode_meta(street, city, state, country)
+            except Exception as e:
+                self._logger.error("Error geocoding", e)
+                lng_lat, metadata = geocoder_error_response("Error geocoding")
+            results.append((cartodb_id, lng_lat, metadata))
+        return results
+
+    def _batch_geocode(self, searches):
+        bulk_results = {}
+        pool = Pool(processes=self.PARALLEL_PROCESSES)
+        for search in searches:
+            (cartodb_id, street, city, state, country) = search
+            address = compose_address(street, city, state, country)
+            if address:
+                components = self._build_optional_parameters(city, state, country)
+                result = pool.apply_async(async_geocoder,
+                                          (self.geocoder, address, components))
+                bulk_results[cartodb_id] = result
+        pool.close()
+        pool.join()
+
+        try:
+            results = []
+            for cartodb_id, bulk_result in bulk_results.items():
+                try:
+                    lng_lat, metadata = self._process_results(bulk_result.get())
+                except Exception as e:
+                    msg = 'Error at Google async_geocoder'
+                    self._logger.error(msg, e)
+                    lng_lat, metadata = geocoder_error_response(msg)
+
+                results.append((cartodb_id, lng_lat, metadata))
+            return results
+        except Exception as e:
+            self._logger.error('General error', exception=e)
+            raise e

server/lib/python/cartodb_services/cartodb_services/google/client_factory.py

@@ -5,6 +5,7 @@ import googlemaps
 import base64
 from exceptions import InvalidGoogleCredentials
 
+
 class GoogleMapsClientFactory():
     clients = {}
 
@@ -13,11 +14,14 @@ class GoogleMapsClientFactory():
         cache_key = "{}:{}:{}".format(client_id, client_secret, channel)
         client = cls.clients.get(cache_key)
         if not client:
-            cls.assert_valid_crendentials(client_secret)
-            client = googlemaps.Client(
-                client_id=client_id,
-                client_secret=client_secret,
-                channel=channel)
+            if client_id:
+                cls.assert_valid_crendentials(client_secret)
+                client = googlemaps.Client(
+                    client_id=client_id,
+                    client_secret=client_secret,
+                    channel=channel)
+            else:
+                client = googlemaps.Client(key=client_secret)
             cls.clients[cache_key] = client
         return client
 

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,16 +1,41 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
-import googlemaps
 from urlparse import parse_qs
 
 from exceptions import MalformedResult
+from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
 from cartodb_services.google.exceptions import InvalidGoogleCredentials
 from client_factory import GoogleMapsClientFactory
 
+PARTIAL_FACTOR = 0.8
+RELEVANCE_BY_LOCATION_TYPE = {
+    'ROOFTOP': 1,
+    'GEOMETRIC_CENTER': 0.9,
+    'RANGE_INTERPOLATED': 0.8,
+    'APPROXIMATE': 0.7
+}
+PRECISION_BY_LOCATION_TYPE = {
+    'ROOFTOP': PRECISION_PRECISE,
+    'GEOMETRIC_CENTER': PRECISION_PRECISE,
+    'RANGE_INTERPOLATED': PRECISION_INTERPOLATED,
+    'APPROXIMATE': PRECISION_INTERPOLATED
+}
+MATCH_TYPE_BY_MATCH_LEVEL = {
+    'point_of_interest': 'point_of_interest',
+    'country': 'country',
+    'administrative_area_level_1': 'state',
+    'administrative_area_level_2': 'county',
+    'locality': 'locality',
+    'sublocality': 'district',
+    'street_address': 'street',
+    'intersection': 'intersection',
+    'street_number': 'street_number',
+    'postal_code': 'postal_code'
+}
 
-class GoogleMapsGeocoder:
-    """A Google Maps Geocoder wrapper for python"""
+
+class GoogleMapsGeocoder():
 
     def __init__(self, client_id, client_secret, logger):
         if client_id is None:
@@ -20,25 +45,49 @@ class GoogleMapsGeocoder:
         self.geocoder = GoogleMapsClientFactory.get(self.client_id, self.client_secret, self.channel)
         self._logger = logger
 
-    def geocode(self, searchtext, city=None, state=None,
-                country=None):
+    def geocode(self, searchtext, city=None, state=None, country=None):
+        return self.geocode_meta(searchtext, city, state, country)[0]
+
+    def geocode_meta(self, searchtext, city=None, state=None, country=None):
+        address = compose_address(searchtext, city, state, country)
         try:
             opt_params = self._build_optional_parameters(city, state, country)
-            results = self.geocoder.geocode(address=searchtext,
+            results = self.geocoder.geocode(address=address,
                                             components=opt_params)
-            if results:
-                return self._extract_lng_lat_from_result(results[0])
-            else:
-                return []
-        except KeyError:
+            return self._process_results(results)
+        except KeyError as e:
+            self._logger.error('address: {}'.format(address), e)
             raise MalformedResult()
 
+    def _process_results(self, results):
+        if results:
+            return [
+                self._extract_lng_lat_from_result(results[0]),
+                self._extract_metadata_from_result(results[0])
+            ]
+        else:
+            return EMPTY_RESPONSE
+
     def _extract_lng_lat_from_result(self, result):
         location = result['geometry']['location']
         longitude = location['lng']
         latitude = location['lat']
         return [longitude, latitude]
 
+    def _extract_metadata_from_result(self, result):
+        location_type = result['geometry']['location_type']
+        base_relevance = RELEVANCE_BY_LOCATION_TYPE[location_type]
+        partial_match = result.get('partial_match', False)
+        partial_factor = PARTIAL_FACTOR if partial_match else 1
+        match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None)
+                       for match_level in result['types']]
+        return geocoder_metadata(
+            base_relevance * partial_factor,
+            PRECISION_BY_LOCATION_TYPE[location_type],
+            filter(None, match_types)
+        )
+
+
     def _build_optional_parameters(self, city=None, state=None,
                                    country=None):
         optional_params = {}

server/lib/python/cartodb_services/cartodb_services/here/__init__.py

@@ -1,2 +1,3 @@
 from geocoder import HereMapsGeocoder
+from bulk_geocoder import HereMapsBulkGeocoder
 from routing import HereMapsRoutingIsoline

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -0,0 +1,152 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+
+import requests, time, zipfile, io, csv, cStringIO
+import xml.etree.ElementTree as ET
+from collections import namedtuple
+from requests.adapters import HTTPAdapter
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.here import HereMapsGeocoder
+from cartodb_services.geocoder import geocoder_metadata, geocoder_error_response
+from cartodb_services.metrics import Traceable
+from cartodb_services.tools.exceptions import ServiceException
+
+
+HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status')
+
+class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 1000000  # From the docs
+    MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
+    BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
+    # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
+    META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
+    MAX_STALLED_RETRIES = 100
+    BATCH_RETRY_SLEEP_S = 5
+    JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
+
+    def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
+        HereMapsGeocoder.__init__(self, app_id, app_code, logger, service_params, maxresults)
+        self.session = requests.Session()
+        self.session.mount(self.BATCH_URL,
+                           HTTPAdapter(max_retries=self.max_retries))
+        self.credentials_params = {
+            'app_id': self.app_id,
+            'app_code': self.app_code,
+        }
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            (search_id, address, city, state, country) = search
+            try:
+                result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
+            except Exception as e:
+                self._logger.error("Error geocoding", e)
+                result = geocoder_error_response("Error geocoding")
+            results.append((search_id, result[0], result[1]))
+        return results
+
+    def _batch_geocode(self, searches):
+        request_id = self._send_batch(self._searches_to_csv(searches))
+
+        last_processed = 0
+        stalled_retries = 0
+        # https://developer.here.com/documentation/batch-geocoder/topics/job-status.html
+        while True:
+            job_info = self._job_status(request_id)
+            if job_info.processed_count == last_processed:
+                stalled_retries += 1
+                if stalled_retries > self.MAX_STALLED_RETRIES:
+                    raise Exception('Too many retries for job {}'.format(request_id))
+            else:
+                stalled_retries = 0
+                last_processed = job_info.processed_count
+
+            if job_info.status in self.JOB_FINAL_STATES:
+                break
+            else:
+                time.sleep(self.BATCH_RETRY_SLEEP_S)
+
+        results = self._download_results(request_id)
+
+        return results
+
+    def _searches_to_csv(self, searches):
+        queue = cStringIO.StringIO()
+        writer = csv.writer(queue, delimiter='|')
+        writer.writerow(['recId', 'searchText', 'country'])
+
+        for search in searches:
+            fields = [search.address, search.city, search.state]
+            search_text = ', '.join(filter(None, fields))
+            row = [s.encode("utf-8") if s else ''
+                   for s in [str(search.id), search_text, search.country]]
+            writer.writerow(row)
+
+        return queue.getvalue()
+
+    def _send_batch(self, data):
+        cols = 'displayLatitude,displayLongitude,' + ','.join(self.META_COLS)
+        request_params = self.credentials_params.copy()
+        request_params.update({
+            'gen': 8,
+            'action': 'run',
+            # 'mailto': 'juanignaciosl@carto.com',
+            'header': 'true',
+            'inDelim': '|',
+            'outDelim': '|',
+            'outCols': cols,
+            'outputcombined': 'true'
+        })
+
+        response = self.session.post(self.BATCH_URL, data=data,
+                                     params=request_params,
+                                     timeout=(self.connect_timeout, self.read_timeout))
+
+        if response.status_code == 200:
+            root = ET.fromstring(response.text)
+            return root.find('./Response/MetaInfo/RequestId').text
+        else:
+            raise ServiceException("Error sending HERE batch", response)
+
+    def _job_status(self, request_id):
+        polling_params = self.credentials_params.copy()
+        polling_params.update({'action': 'status'})
+        polling_r = self.session.get("{}/{}".format(self.BATCH_URL, request_id),
+                                params=polling_params,
+                                timeout=(self.connect_timeout, self.read_timeout))
+        polling_root = ET.fromstring(polling_r.text)
+        return HereJobStatus(
+            total_count=int(polling_root.find('./Response/TotalCount').text),
+            processed_count=int(polling_root.find('./Response/ProcessedCount').text),
+            status=polling_root.find('./Response/Status').text)
+
+    def _download_results(self, job_id):
+        result_r = self.session.get("{}/{}/result".format(self.BATCH_URL, job_id),
+                                    params=self.credentials_params,
+                                    timeout=(self.connect_timeout, self.read_timeout))
+        root_zip = zipfile.ZipFile(io.BytesIO(result_r.content))
+
+        results = []
+        for name in root_zip.namelist():
+            if name.endswith('_out.txt'):
+                reader = csv.DictReader(root_zip.open(name), delimiter='|')
+                for row in reader:
+                    if row['SeqNumber'] == '1':  # First per requested data
+                        precision = self.PRECISION_BY_MATCH_TYPE[
+                            row.get('matchType', 'pointAddress')]
+                        match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(row['matchLevel'], None)
+                        results.append((row['recId'],
+                                        [row['displayLongitude'], row['displayLatitude']],
+                                        geocoder_metadata(
+                                            float(row['relevance']),
+                                            precision,
+                                            [match_type] if match_type else []
+                                        )))
+
+        return results
+

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -6,9 +6,9 @@ import requests
 
 from requests.adapters import HTTPAdapter
 from exceptions import *
+from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE
 from cartodb_services.metrics import Traceable
 
-
 class HereMapsGeocoder(Traceable):
     'A Here Maps Geocoder wrapper for python'
 
@@ -52,6 +52,23 @@ class HereMapsGeocoder(Traceable):
         'strictlanguagemode'
         ] + ADDRESS_PARAMS
 
+    PRECISION_BY_MATCH_TYPE = {
+        'pointAddress': PRECISION_PRECISE,
+        'interpolated': PRECISION_INTERPOLATED
+    }
+    MATCH_TYPE_BY_MATCH_LEVEL = {
+        'landmark': 'point_of_interest',
+        'country': 'country',
+        'state': 'state',
+        'county': 'county',
+        'city': 'locality',
+        'district': 'district',
+        'street': 'street',
+        'intersection': 'intersection',
+        'houseNumber': 'street_number',
+        'postalCode': 'postal_code'
+    }
+
     def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS):
         service_params = service_params or {}
         self.app_id = app_id
@@ -65,12 +82,15 @@ class HereMapsGeocoder(Traceable):
         self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
 
     def geocode(self, **kwargs):
+        return self.geocode_meta(**kwargs)[0]
+
+    def geocode_meta(self, **kwargs):
         params = {}
         for key, value in kwargs.iteritems():
             if value and value.strip():
                 params[key] = value
         if not params:
-            return []
+            return EMPTY_RESPONSE
         return self._execute_geocode(params)
 
     def _execute_geocode(self, params):
@@ -78,11 +98,13 @@ class HereMapsGeocoder(Traceable):
             raise BadGeocodingParams(params)
         try:
             response = self._perform_request(params)
-            results = response['Response']['View'][0]['Result'][0]
-            return self._extract_lng_lat_from_result(results)
+            result = response['Response']['View'][0]['Result'][0]
+            return [self._extract_lng_lat_from_result(result),
+                    self._extract_metadata_from_result(result)]
         except IndexError:
-            return []
-        except KeyError:
+            return EMPTY_RESPONSE
+        except KeyError as e:
+            self._logger.error('params: {}'.format(params), e)
             raise MalformedResult()
 
     def _perform_request(self, params):
@@ -105,7 +127,7 @@ class HereMapsGeocoder(Traceable):
             self._logger.warning('Error 4xx trying to geocode street using HERE',
                                data={"response": response.json(), "params":
                                      params})
-            return []
+            return EMPTY_RESPONSE
         else:
             self._logger.error('Error trying to geocode street using HERE',
                                data={"response": response.json(), "params":
@@ -118,3 +140,14 @@ class HereMapsGeocoder(Traceable):
         latitude = location['DisplayPosition']['Latitude']
 
         return [longitude, latitude]
+
+    def _extract_metadata_from_result(self, result):
+        # See https://stackoverflow.com/questions/51285622/missing-matchtype-at-here-geocoding-responses
+        precision = self.PRECISION_BY_MATCH_TYPE[
+            result.get('MatchType', 'pointAddress')]
+        match_type = self.MATCH_TYPE_BY_MATCH_LEVEL.get(result['MatchLevel'], None)
+        return geocoder_metadata(
+            result['Relevance'],
+            precision,
+            [match_type] if match_type else []
+        )

server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py

@@ -1,4 +1,5 @@
 from routing import MapboxRouting, MapboxRoutingResponse
 from geocoder import MapboxGeocoder
+from bulk_geocoder import MapboxBulkGeocoder
 from isolines import MapboxIsolines, MapboxIsochronesResponse
 from matrix_client import MapboxMatrixClient

server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py

@@ -0,0 +1,65 @@
+import requests
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.mapbox import MapboxGeocoder
+from iso3166 import countries
+from cartodb_services.tools.country import country_to_iso3
+
+
+class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 50  # From the docs
+    MIN_BATCHED_SEARCH = 0
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    def __init__(self, token, logger, service_params=None):
+        MapboxGeocoder.__init__(self, token, logger, service_params)
+        self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
+        self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
+        self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
+        self.session = requests.Session()
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            elements = self._encoded_elements(search)
+            result = self.geocode_meta(*elements)
+
+            results.append((search[0], result[0], result[1]))
+        return results
+
+    def _encoded_elements(self, search):
+        (search_id, address, city, state, country) = search
+        address = address.encode('utf-8') if address else None
+        city = city.encode('utf-8') if city else None
+        state = state.encode('utf-8') if state else None
+        country = self._country_code(country) if country else None
+        return address, city, state, country
+
+    def _batch_geocode(self, searches):
+        if len(searches) == 1:
+            return self._serial_geocode(searches)
+        else:
+            frees = []
+            for search in searches:
+                elements = self._encoded_elements(search)
+                free = ', '.join([elem for elem in elements if elem])
+                frees.append(free)
+
+            full_results = self.geocode_free_text_meta(frees)
+            results = []
+            for s, r in zip(searches, full_results):
+                results.append((s[0], r[0], r[1]))
+            return results
+
+    def _country_code(self, country):
+        country_iso3166 = None
+        country_iso3 = country_to_iso3(country)
+        if country_iso3:
+            country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+        return country_iso3166
+

server/lib/python/cartodb_services/cartodb_services/mapbox/geocoder.py

@@ -5,6 +5,7 @@ Python client for the Mapbox Geocoder service.
 import json
 import requests
 from mapbox import Geocoder
+from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE, EMPTY_BATCH_RESPONSE, TOO_MANY_REQUESTS_ERROR_RESPONSE, geocoder_error_response
 from cartodb_services.metrics import Traceable
 from cartodb_services.tools.exceptions import ServiceException
 from cartodb_services.tools.qps import qps_retry
@@ -22,6 +23,17 @@ ENTRY_COORDINATES = 'coordinates'
 ENTRY_TYPE = 'type'
 TYPE_POINT = 'Point'
 
+MATCH_TYPE_BY_MATCH_LEVEL = {
+    'poi': 'point_of_interest',
+    'poi.landmark': 'point_of_interest',
+    'place': 'point_of_interest',
+    'country': 'country',
+    'region': 'state',
+    'locality': 'locality',
+    'district': 'district',
+    'address': 'street'
+}
+
 
 class MapboxGeocoder(Traceable):
     '''
@@ -40,18 +52,24 @@ class MapboxGeocoder(Traceable):
     def _parse_geocoder_response(self, response):
         json_response = json.loads(response)
 
-        # If Mapbox returns more that one result, take the first one
         if json_response:
-            if type(json_response) == list:
-                json_response = json_response[0]
+            if type(json_response) != list:
+                json_response = [json_response]
 
-            if json_response[ENTRY_FEATURES]:
-                feature = json_response[ENTRY_FEATURES][0]
-                return self._extract_lng_lat_from_feature(feature)
-            else:
-                return []
+            result = []
+            for a_json_response in json_response:
+                if a_json_response[ENTRY_FEATURES]:
+                    feature = a_json_response[ENTRY_FEATURES][0]
+                    result.append([
+                        self._extract_lng_lat_from_feature(feature),
+                        self._extract_metadata_from_result(feature)
+                        ]
+                    )
+                else:
+                    result.append(EMPTY_RESPONSE)
+            return result
         else:
-            return []
+            return EMPTY_BATCH_RESPONSE
 
     def _extract_lng_lat_from_feature(self, feature):
         geometry = feature[ENTRY_GEOMETRY]
@@ -64,42 +82,114 @@ class MapboxGeocoder(Traceable):
         latitude = location[1]
         return [longitude, latitude]
 
+    def _extract_metadata_from_result(self, result):
+        if result[ENTRY_GEOMETRY].get('interpolated', False):
+            precision = PRECISION_INTERPOLATED
+        else:
+            precision = PRECISION_PRECISE
+
+        match_types = [MATCH_TYPE_BY_MATCH_LEVEL.get(match_level, None)
+                       for match_level in result['place_type']]
+        return geocoder_metadata(
+            self._normalize_relevance(float(result['relevance'])),
+            precision,
+            filter(None, match_types)
+        )
+
+    def _normalize_relevance(self, relevance):
+        return 1 if relevance >= 0.99 else relevance
+
+    def _validate_input(self, searchtext, city=None, state_province=None,
+                        country=None):
+        if searchtext and searchtext.strip():
+            return True
+        elif city:
+            return True
+        elif state_province:
+            return True
+
+        return False
+
     @qps_retry(qps=10)
     def geocode(self, searchtext, city=None, state_province=None,
                 country=None):
-        if searchtext and searchtext.strip():
-            address = [normalize(searchtext)]
-            if city:
-                address.append(normalize(city))
-            if state_province:
-                address.append(normalize(state_province))
+        """
+        :param searchtext:
+        :param city:
+        :param state_province:
+        :param country: Country ISO 3166 code
+        :return: [x, y] on success, raises ServiceException on error
+        """
+        response = self.geocode_meta(searchtext, city, state_province, country)
+        if response:
+            error_message = response[1].get('error', None)
+            if error_message:
+                raise ServiceException(error_message, None)
+            else:
+                return response[0]
         else:
-            return []
+            return EMPTY_RESPONSE
 
+    @qps_retry(qps=10)
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                country=None):
+        if not self._validate_input(searchtext, city, state_province, country):
+            return EMPTY_RESPONSE
+
+        address = []
+        if searchtext and searchtext.strip():
+            address.append(normalize(searchtext))
+        if city:
+            address.append(normalize(city))
+        if state_province:
+            address.append(normalize(state_province))
+
+        free_search = ', '.join(address)
+
+        response = self.geocode_free_text_meta([free_search], country)
+        return response[0] if response else EMPTY_RESPONSE
+
+    @qps_retry(qps=10)
+    def geocode_free_text_meta(self, free_searches, country=None):
+        """
+        :param free_searches: Free text searches
+        :param country: Country ISO 3166 code
+        :return: list of [x, y] on success, [] on error
+        """
         country = [country] if country else None
 
         try:
-            response = self._geocoder.forward(address=', '.join(address).decode('utf-8'),
-                                              country=country,
-                                              limit=1)
-
+            free_search = ';'.join([self._escape(fs) for fs in free_searches])
+            response = self._geocoder.forward(address=free_search.decode('utf-8'),
+                                              limit=1,
+                                              country=country)
             if response.status_code == requests.codes.ok:
                 return self._parse_geocoder_response(response.text)
+            elif response.status_code == requests.codes.too_many_requests:
+                return [TOO_MANY_REQUESTS_ERROR_RESPONSE] * len(free_searches)
             elif response.status_code == requests.codes.bad_request:
-                return []
+                return EMPTY_BATCH_RESPONSE
             elif response.status_code == requests.codes.unprocessable_entity:
-                return []
+                return EMPTY_BATCH_RESPONSE
             else:
-                raise ServiceException(response.status_code, response)
+                msg = "Unkown status: {}".format(response.status_code)
+                self._logger.warning(msg, data={"searches": free_searches})
+                return [geocoder_error_response(msg)] * len(free_searches)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down
-            self._logger.error('Timeout connecting to Mapbox geocoding server',
-                               te)
-            raise ServiceException('Error geocoding {0} using Mapbox'.format(
-                searchtext), None)
+            msg = 'Timeout connecting to Mapbox geocoding server'
+            self._logger.error(msg, te)
+            return [geocoder_error_response(msg)] * len(free_searches)
         except requests.ConnectionError as ce:
             # Don't raise the exception to continue with the geocoding job
             self._logger.error('Error connecting to Mapbox geocoding server',
                                exception=ce)
-            return []
+            return EMPTY_BATCH_RESPONSE
+
+    def _escape(self, free_search):
+        # Semicolon is used to separate batch geocoding; there's no documented
+        # way to pass actual semicolons, and %3B or &#59; won't work (check
+        # TestBulkStreetFunctions.test_semicolon and the docs,
+        # https://www.mapbox.com/api-documentation/#batch-requests)
+        return free_search.replace(';', ',')

server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py

@@ -4,6 +4,7 @@ Uses the Mapbox Time Matrix service.
 '''
 
 import json
+from cartodb_services.tools import Coordinate
 from cartodb_services.tools.spherical import (get_angles,
                                               calculate_dest_location)
 from cartodb_services.mapbox.matrix_client import (validate_profile,
@@ -11,7 +12,9 @@ from cartodb_services.mapbox.matrix_client import (validate_profile,
                                                    PROFILE_WALKING,
                                                    PROFILE_DRIVING,
                                                    PROFILE_CYCLING,
-                                                   ENTRY_DURATIONS)
+                                                   ENTRY_DURATIONS,
+                                                   ENTRY_DESTINATIONS,
+                                                   ENTRY_LOCATION)
 
 MAX_SPEEDS = {
     PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
@@ -53,6 +56,7 @@ class MapboxIsolines():
             return []
 
         costs = [None] * number_of_angles
+        destinations = [None] * number_of_angles
 
         for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]):
             if cost:
@@ -60,7 +64,11 @@ class MapboxIsolines():
             else:
                 costs[idx] = isorange
 
-        return costs
+        for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]):
+            destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0],
+                                           destination[ENTRY_LOCATION][1])
+
+        return costs, destinations
 
     def calculate_isochrone(self, origin, time_ranges,
                             profile=DEFAULT_PROFILE):
@@ -122,10 +130,12 @@ class MapboxIsolines():
             # NOTE: sometimes it cannot calculate the cost and returns None.
             #   Just assume isorange and stop the calculations there
 
-            costs = cost_method(origin=origin, targets=location_estimates,
-                                isorange=isorange, profile=profile,
-                                unit_factor=unit_factor,
-                                number_of_angles=number_of_angles)
+            costs, destinations = cost_method(origin=origin,
+                                              targets=location_estimates,
+                                              isorange=isorange,
+                                              profile=profile,
+                                              unit_factor=unit_factor,
+                                              number_of_angles=number_of_angles)
 
             if not costs:
                 continue
@@ -152,8 +162,8 @@ class MapboxIsolines():
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c != isorange:
-                location_estimates_filtered.append(location_estimates[i])
+            if c != isorange and c < isorange * (1 + tolerance):
+                location_estimates_filtered.append(destinations[i])
 
         return location_estimates_filtered
 

server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py

@@ -30,6 +30,8 @@ VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC,
                   PROFILE_WALKING]
 
 ENTRY_DURATIONS = 'durations'
+ENTRY_DESTINATIONS = 'destinations'
+ENTRY_LOCATION = 'location'
 
 
 def validate_profile(profile):

server/lib/python/cartodb_services/cartodb_services/mapzen/types.py

@@ -28,7 +28,7 @@ def coordinates_to_polygon(coordinates):
     wkt_coordinates = ','.join(result_coordinates)
 
     try:
-        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates)
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3)) as geom".format(wkt_coordinates)
         geometry = plpy.execute(sql, 1)[0]['geom']
     except BaseException as e:
         plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e))

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
+from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService
 from log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -86,28 +86,6 @@ class DataObservatoryConfig(ServiceConfig):
         return 'data observatory'
 
 
-class ObservatorySnapshotConfig(DataObservatoryConfig):
-
-    SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
-    QUOTA_KEY = 'obs_snapshot_quota'
-    PERIOD_END_DATE = 'period_end_date'
-
-    def __init__(self, redis_connection, db_conn, username, orgname=None):
-        super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
-                                            username, orgname)
-        self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
-        if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
-            self._soft_limit = True
-        else:
-            self._soft_limit = False
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-        self._connection_str = self._db_config.data_observatory_connection_str
-
-    @property
-    def service_type(self):
-        return 'obs_snapshot'
-
-
 class ObservatoryConfig(DataObservatoryConfig):
 
     SOFT_LIMIT_KEY = 'soft_obs_general_limit'
@@ -136,7 +114,8 @@ class RoutingConfig(ServiceConfig):
     ROUTING_PROVIDER_KEY = 'routing_provider'
     MAPZEN_PROVIDER = 'mapzen'
     MAPBOX_PROVIDER = 'mapbox'
-    DEFAULT_PROVIDER = MAPZEN_PROVIDER
+    TOMTOM_PROVIDER = 'tomtom'
+    DEFAULT_PROVIDER = MAPBOX_PROVIDER
     QUOTA_KEY = 'mapzen_routing_quota'
     SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
     METRICS_LOG_KEY = 'routing_log_path'
@@ -147,11 +126,16 @@ class RoutingConfig(ServiceConfig):
         self._routing_provider = self._redis_config[self.ROUTING_PROVIDER_KEY]
         if not self._routing_provider:
             self._routing_provider = self.DEFAULT_PROVIDER
-        self._mapzen_api_key = self._db_config.mapzen_routing_api_key
-        self._mapzen_service_params = self._db_config.mapzen_routing_service_params
-        self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
-        self._mapbox_service_params = self._db_config.mapbox_routing_service_params
-        self._set_monthly_quota()
+        if self._routing_provider == self.MAPZEN_PROVIDER:
+            self._mapzen_api_key = self._db_config.mapzen_routing_api_key
+            self._mapzen_service_params = self._db_config.mapzen_routing_service_params
+        elif self._routing_provider == self.MAPBOX_PROVIDER:
+            self._mapbox_api_keys = self._db_config.mapbox_routing_api_keys
+            self._mapbox_service_params = self._db_config.mapbox_routing_service_params
+        elif self._routing_provider == self.TOMTOM_PROVIDER:
+            self._tomtom_api_keys = self._db_config.tomtom_routing_api_keys
+            self._tomtom_service_params = self._db_config.tomtom_routing_service_params
+        self._routing_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._set_soft_limit()
         self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
 
@@ -161,6 +145,8 @@ class RoutingConfig(ServiceConfig):
             return 'routing_mapzen'
         elif self._routing_provider == self.MAPBOX_PROVIDER:
             return 'routing_mapbox'
+        elif self._routing_provider == self.TOMTOM_PROVIDER:
+            return 'routing_tomtom'
 
     @property
     def provider(self):
@@ -190,9 +176,25 @@ class RoutingConfig(ServiceConfig):
     def mapbox_service_params(self):
         return self._mapbox_service_params
 
+    @property
+    def tomtom_provider(self):
+        return self._routing_provider == self.TOMTOM_PROVIDER
+
+    @property
+    def tomtom_api_keys(self):
+        return self._tomtom_api_keys
+
+    @property
+    def tomtom_service_params(self):
+        return self._tomtom_service_params
+
+    @property
+    def routing_quota(self):
+        return self._routing_quota
+
     @property
     def monthly_quota(self):
-        return self._monthly_quota
+        return self._routing_quota
 
     @property
     def period_end_date(self):
@@ -202,9 +204,6 @@ class RoutingConfig(ServiceConfig):
     def soft_limit(self):
         return self._soft_limit
 
-    def _set_monthly_quota(self):
-        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
-
     def _set_soft_limit(self):
         if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_limit = True
@@ -225,8 +224,9 @@ class IsolinesRoutingConfig(ServiceConfig):
     GEOCODER_PROVIDER_KEY = 'geocoder_provider'
     MAPZEN_PROVIDER = 'mapzen'
     MAPBOX_PROVIDER = 'mapbox'
+    TOMTOM_PROVIDER = 'tomtom'
     HEREMAPS_PROVIDER = 'heremaps'
-    DEFAULT_PROVIDER = MAPZEN_PROVIDER
+    DEFAULT_PROVIDER = MAPBOX_PROVIDER
     METRICS_LOG_KEY = 'isolines_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -258,6 +258,9 @@ class IsolinesRoutingConfig(ServiceConfig):
             self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys
             self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
             self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
+        elif self._isolines_provider == self.TOMTOM_PROVIDER:
+            self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys
+            self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params
 
     @property
     def service_type(self):
@@ -267,6 +270,8 @@ class IsolinesRoutingConfig(ServiceConfig):
             return 'mapzen_isolines'
         elif self._isolines_provider == self.MAPBOX_PROVIDER:
             return 'mapbox_isolines'
+        elif self._isolines_provider == self.TOMTOM_PROVIDER:
+            return 'tomtom_isolines'
 
     @property
     def google_services_user(self):
@@ -328,6 +333,18 @@ class IsolinesRoutingConfig(ServiceConfig):
     def mapbox_provider(self):
         return self._isolines_provider == self.MAPBOX_PROVIDER
 
+    @property
+    def tomtom_isolines_api_keys(self):
+        return self._tomtom_isolines_api_keys
+
+    @property
+    def tomtom_isolines_service_params(self):
+        return self._tomtom_isolines_service_params
+
+    @property
+    def tomtom_provider(self):
+        return self._isolines_provider == self.TOMTOM_PROVIDER
+
     @property
     def heremaps_provider(self):
         return self._isolines_provider == self.HEREMAPS_PROVIDER
@@ -386,12 +403,14 @@ class GeocoderConfig(ServiceConfig):
     GEOCODER_PROVIDER = 'geocoder_provider'
     MAPBOX_GEOCODER = 'mapbox'
     MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
+    TOMTOM_GEOCODER = 'tomtom'
+    TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
     QUOTA_KEY = 'geocoding_quota'
     SOFT_LIMIT_KEY = 'soft_geocoding_limit'
     USERNAME_KEY = 'username'
     ORGNAME_KEY = 'orgname'
     PERIOD_END_DATE = 'period_end_date'
-    DEFAULT_PROVIDER = MAPZEN_GEOCODER
+    DEFAULT_PROVIDER = MAPBOX_GEOCODER
     METRICS_LOG_KEY = 'geocoder_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
@@ -415,6 +434,9 @@ class GeocoderConfig(ServiceConfig):
         elif self._geocoder_provider == self.MAPBOX_GEOCODER:
             if not self.mapbox_api_keys:
                 raise ConfigException("""Mapbox config is not set up""")
+        elif self._geocoder_provider == self.TOMTOM_GEOCODER:
+            if not self.tomtom_api_keys:
+                raise ConfigException("""TomTom config is not set up""")
 
         return True
 
@@ -450,6 +472,10 @@ class GeocoderConfig(ServiceConfig):
             self._mapbox_api_keys = db_config.mapbox_geocoder_api_keys
             self._cost_per_hit = 0
             self._mapbox_service_params = db_config.mapbox_geocoder_service_params
+        elif self._geocoder_provider == self.TOMTOM_GEOCODER:
+            self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
+            self._cost_per_hit = 0
+            self._tomtom_service_params = db_config.tomtom_geocoder_service_params
 
     @property
     def service_type(self):
@@ -459,6 +485,8 @@ class GeocoderConfig(ServiceConfig):
             return 'geocoder_mapzen'
         elif self._geocoder_provider == self.MAPBOX_GEOCODER:
             return 'geocoder_mapbox'
+        elif self._geocoder_provider == self.TOMTOM_GEOCODER:
+            return 'geocoder_tomtom'
         elif self._geocoder_provider == self.NOKIA_GEOCODER:
             return 'geocoder_here'
 
@@ -478,6 +506,10 @@ class GeocoderConfig(ServiceConfig):
     def mapbox_geocoder(self):
         return self._geocoder_provider == self.MAPBOX_GEOCODER
 
+    @property
+    def tomtom_geocoder(self):
+        return self._geocoder_provider == self.TOMTOM_GEOCODER
+
     @property
     def google_client_id(self):
         return self._google_maps_client_id
@@ -529,6 +561,14 @@ class GeocoderConfig(ServiceConfig):
     def mapbox_service_params(self):
         return self._mapbox_service_params
 
+    @property
+    def tomtom_api_keys(self):
+        return self._tomtom_api_keys
+
+    @property
+    def tomtom_service_params(self):
+        return self._tomtom_service_params
+
     @property
     def is_high_resolution(self):
         return True
@@ -559,6 +599,7 @@ class ServicesDBConfig:
         self._get_here_config()
         self._get_mapzen_config()
         self._get_mapbox_config()
+        self._get_tomtom_config()
         self._get_data_observatory_config()
 
     def _get_server_config(self):
@@ -576,50 +617,67 @@ class ServicesDBConfig:
         heremaps_conf_json = self._get_conf('heremaps_conf')
         if not heremaps_conf_json:
             raise ConfigException('Here maps configuration missing')
-        else:
-            heremaps_conf = json.loads(heremaps_conf_json)
-            self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
-            self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
-            self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
-                'geocoder_cost_per_hit']
-            self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
-            self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
-            self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
-            self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
+
+        heremaps_conf = json.loads(heremaps_conf_json)
+        self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
+        self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
+        self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
+            'geocoder_cost_per_hit']
+        self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {})
+        self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
+        self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
+        self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {})
 
     def _get_mapzen_config(self):
         mapzen_conf_json = self._get_conf('mapzen_conf')
+        # We dont use mapzen anymore so we don't need to check for its configuration
         if not mapzen_conf_json:
-            raise ConfigException('Mapzen configuration missing')
-        else:
-            mapzen_conf = json.loads(mapzen_conf_json)
-            self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
-            self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
-            self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
-            self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
-            self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
-            self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
-            self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
-            self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
-            self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
-            self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
+            return
+
+        mapzen_conf = json.loads(mapzen_conf_json)
+        self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
+        self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
+        self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {})
+        self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
+        self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
+        self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
+        self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {})
+        self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
+        self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
+        self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
 
     def _get_mapbox_config(self):
         mapbox_conf_json = self._get_conf('mapbox_conf')
         if not mapbox_conf_json:
             raise ConfigException('Mapbox configuration missing')
+
+        mapbox_conf = json.loads(mapbox_conf_json)
+        self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
+        self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
+        self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
+        self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
+        self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
+        self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
+        self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
+        self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys']
+        self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
+        self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
+
+    def _get_tomtom_config(self):
+        tomtom_conf_json = self._get_conf('tomtom_conf')
+        if not tomtom_conf_json:
+            raise ConfigException('TomTom configuration missing')
         else:
-            mapbox_conf = json.loads(mapbox_conf_json)
-            self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys']
-            self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota']
-            self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {})
-            self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {})
-            self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys']
-            self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota']
-            self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {})
-            self._mapbox_geocoder_api_keys = mapbox_conf['geocoder']['api_keys']
-            self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota']
-            self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {})
+            tomtom_conf = json.loads(tomtom_conf_json)
+            self._tomtom_isolines_api_keys = tomtom_conf['isolines']['api_keys']
+            self._tomtom_isolines_quota = tomtom_conf['isolines']['monthly_quota']
+            self._tomtom_isolines_service_params = tomtom_conf.get('isolines', {}).get('service', {})
+            self._tomtom_routing_api_keys = tomtom_conf['routing']['api_keys']
+            self._tomtom_routing_quota = tomtom_conf['routing']['monthly_quota']
+            self._tomtom_routing_service_params = tomtom_conf['routing'].get('service', {})
+            self._tomtom_geocoder_api_keys = tomtom_conf['geocoder']['api_keys']
+            self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
+            self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})
 
     def _get_data_observatory_config(self):
         do_conf_json = self._get_conf('data_observatory_conf')
@@ -754,6 +812,42 @@ class ServicesDBConfig:
     def mapbox_geocoder_service_params(self):
         return self._mapbox_geocoder_service_params
 
+    @property
+    def tomtom_isolines_api_keys(self):
+        return self._tomtom_isolines_api_keys
+
+    @property
+    def tomtom_isolines_monthly_quota(self):
+        return self._tomtom_isolines_quota
+
+    @property
+    def tomtom_isolines_service_params(self):
+        return self._tomtom_isolines_service_params
+
+    @property
+    def tomtom_routing_api_keys(self):
+        return self._tomtom_routing_api_keys
+
+    @property
+    def tomtom_routing_monthly_quota(self):
+        return self._tomtom_routing_quota
+
+    @property
+    def tomtom_routing_service_params(self):
+        return self._tomtom_routing_service_params
+
+    @property
+    def tomtom_geocoder_api_keys(self):
+        return self._tomtom_geocoder_api_keys
+
+    @property
+    def tomtom_geocoder_monthly_quota(self):
+        return self._tomtom_geocoder_quota
+
+    @property
+    def tomtom_geocoder_service_params(self):
+        return self._tomtom_geocoder_service_params
+
     @property
     def data_observatory_connection_str(self):
         return self._data_observatory_connection_str
@@ -774,7 +868,6 @@ class ServicesRedisConfig:
     QUOTA_KEY = 'geocoding_quota'
     ISOLINES_QUOTA_KEY = 'here_isolines_quota'
     ROUTING_QUOTA_KEY = 'mapzen_routing_quota'
-    OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
     OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
     PERIOD_END_DATE = 'period_end_date'
     GEOCODER_PROVIDER_KEY = 'geocoder_provider'
@@ -818,8 +911,6 @@ class ServicesRedisConfig:
                 user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
             if self.ROUTING_QUOTA_KEY in org_config:
                 user_config[self.ROUTING_QUOTA_KEY] = org_config[self.ROUTING_QUOTA_KEY]
-            if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
-                user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
             if self.OBS_GENERAL_QUOTA_KEY in org_config:
                 user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
             if self.PERIOD_END_DATE in org_config:

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -69,25 +69,16 @@ class QuotaChecker:
 
     def check(self):
         """ Check if the current user quota surpasses the current quota """
-        if re.match('geocoder_*',
+        if re.match('^geocoder_',
                     self._user_service_config.service_type) is not None:
             return self.__check_geocoder_quota()
-        elif re.match('here_isolines',
+        elif re.match('.*_isolines$',
                       self._user_service_config.service_type) is not None:
             return self.__check_isolines_quota()
-        elif re.match('mapzen_isolines',
-                      self._user_service_config.service_type) is not None:
-            return self.__check_isolines_quota()
-        elif re.match('mapbox_isolines',
-                      self._user_service_config.service_type) is not None:
-            return self.__check_isolines_quota()
-        elif re.match('routing_mapzen',
+        elif re.match('^routing_',
                       self._user_service_config.service_type) is not None:
             return self.__check_routing_quota()
-        elif re.match('routing_mapbox',
-                      self._user_service_config.service_type) is not None:
-            return self.__check_routing_quota()
-        elif re.match('obs_*',
+        elif re.match('^obs_',
                       self._user_service_config.service_type) is not None:
             return self.__check_data_observatory_quota()
         else:
@@ -122,7 +113,7 @@ class QuotaChecker:
             return False
 
     def __check_routing_quota(self):
-        user_quota = self._user_service_config.monthly_quota
+        user_quota = self._user_service_config.routing_quota
         today = date.today()
         service_type = self._user_service_config.service_type
         current_used = self._user_service.used_quota(service_type, today)

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -22,8 +22,10 @@ class UserMetricsService:
     SERVICE_HERE_ISOLINES = 'here_isolines'
     SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
     SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
+    SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines'
     SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
+    SERVICE_TOMTOM_ROUTING = 'routing_tomtom'
     SERVICE_OBSERVATORY = 'obs_general'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
@@ -36,10 +38,12 @@ class UserMetricsService:
     def used_quota(self, service_type, date):
         if service_type in [self.SERVICE_HERE_ISOLINES,
                             self.SERVICE_MAPZEN_ISOLINES,
-                            self.SERVICE_MAPBOX_ISOLINES]:
+                            self.SERVICE_MAPBOX_ISOLINES,
+                            self.SERVICE_TOMTOM_ISOLINES]:
             return self.__used_isolines_quota(service_type, date)
         elif service_type in [self.SERVICE_MAPZEN_ROUTING,
-                              self.SERVICE_MAPBOX_ROUTING]:
+                              self.SERVICE_MAPBOX_ROUTING,
+                              self.SERVICE_TOMTOM_ROUTING]:
             return self.__used_routing_quota(service_type, date)
         elif service_type == self.SERVICE_OBSERVATORY:
             return self.__used_observatory_quota(service_type, date)

server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_geocoder_config.py

@@ -0,0 +1,128 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.tomtom.types import TOMTOM_GEOCODER_APIKEY_ROUNDROBIN
+
+
+class TomTomGeocoderConfig(object):
+    """
+    Configuration needed to operate the TomTom geocoder service.
+    """
+
+    def __init__(self,
+                 geocoding_quota,
+                 soft_geocoding_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 tomtom_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._geocoding_quota = geocoding_quota
+        self._soft_geocoding_limit = soft_geocoding_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._tomtom_api_keys = tomtom_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'geocoder_tomtom'
+
+    @property
+    def provider(self):
+        return 'tomtom'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def tomtom_api_key(self):
+        return round_robin(self._tomtom_api_keys, self._GD,
+                           TOMTOM_GEOCODER_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+    # TODO: for BW compat, remove
+    @property
+    def google_geocoder(self):
+        return False
+
+
+class TomTomGeocoderConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        tomtom_server_conf = self._server_conf.get('tomtom_conf')
+        tomtom_api_keys = tomtom_server_conf['geocoder']['api_keys']
+        tomtom_service_params = tomtom_server_conf['geocoder'].get('service', {})
+
+        geocoding_quota = self._get_quota()
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('geocoder_log_path', None)
+
+        return TomTomGeocoderConfig(geocoding_quota,
+                                    soft_geocoding_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    tomtom_api_keys,
+                                    self._username,
+                                    self._orgname,
+                                    tomtom_service_params,
+                                    self._GD)
+
+    def _get_quota(self):
+        geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
+        if geocoding_quota is '':
+            return 0
+
+        return int(geocoding_quota)

server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_isolines_config.py

@@ -0,0 +1,123 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.tomtom.types import TOMTOM_ISOLINES_APIKEY_ROUNDROBIN
+
+
+class TomTomIsolinesConfig(object):
+    """
+    Configuration needed to operate the TomTom directions service.
+    """
+
+    def __init__(self,
+                 isolines_quota,
+                 soft_isolines_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 tomtom_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._isolines_quota = isolines_quota
+        self._soft_isolines_limit = soft_isolines_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._tomtom_api_keys = tomtom_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'tomtom_isolines'
+
+    @property
+    def provider(self):
+        return 'tomtom'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def isolines_quota(self):
+        return self._isolines_quota
+
+    @property
+    def soft_isolines_limit(self):
+        return self._soft_isolines_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def tomtom_api_key(self):
+        return round_robin(self._tomtom_api_keys, self._GD,
+                           TOMTOM_ISOLINES_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+
+class TomTomIsolinesConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        tomtom_server_conf = self._server_conf.get('tomtom_conf')
+        tomtom_api_keys = tomtom_server_conf['isolines']['api_keys']
+        tomtom_service_params = tomtom_server_conf['isolines'].get('service', {})
+
+        isolines_quota = self._get_quota()
+        soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('isolines_log_path', None)
+
+        return TomTomIsolinesConfig(isolines_quota,
+                                    soft_isolines_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    tomtom_api_keys,
+                                    self._username,
+                                    self._orgname,
+                                    tomtom_service_params,
+                                    self._GD)
+
+    def _get_quota(self):
+        isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota')
+        if isolines_quota is '':
+            return 0
+
+        return int(isolines_quota)

server/lib/python/cartodb_services/cartodb_services/refactor/service/tomtom_routing_config.py

@@ -0,0 +1,131 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.tomtom.types import TOMTOM_ROUTING_APIKEY_ROUNDROBIN
+
+
+class TomTomRoutingConfig(object):
+    """
+    Configuration needed to operate the TomTom directions service.
+    """
+
+    def __init__(self,
+                 routing_quota,
+                 soft_routing_limit,
+                 monthly_quota,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 tomtom_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._routing_quota = routing_quota
+        self._soft_routing_limit = soft_routing_limit
+        self._monthly_quota = monthly_quota
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._tomtom_api_keys = tomtom_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'routing_tomtom'
+
+    @property
+    def provider(self):
+        return 'tomtom'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def routing_quota(self):
+        return self._routing_quota
+
+    @property
+    def soft_limit(self):
+        return self._soft_routing_limit
+
+    @property
+    def monthly_quota(self):
+        return self._monthly_quota
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def tomtom_api_key(self):
+        return round_robin(self._tomtom_api_keys, self._GD,
+                           TOMTOM_ROUTING_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+
+class TomTomRoutingConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        tomtom_server_conf = self._server_conf.get('tomtom_conf')
+        tomtom_api_keys = tomtom_server_conf['routing']['api_keys']
+        monthly_quota = tomtom_server_conf['routing']['monthly_quota']
+        tomtom_service_params = tomtom_server_conf['routing'].get('service', {})
+
+        routing_quota = self._get_quota()
+        soft_routing_limit = self._user_conf.get('soft_mapzen_routing_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('routing_log_path', None)
+
+        return TomTomRoutingConfig(routing_quota,
+                                   soft_routing_limit,
+                                   monthly_quota,
+                                   period_end_date,
+                                   cost_per_hit,
+                                   log_path,
+                                   tomtom_api_keys,
+                                   self._username,
+                                   self._orgname,
+                                   tomtom_service_params,
+                                   self._GD)
+
+    def _get_quota(self):
+        routing_quota = self._org_conf.get('mapzen_routing_quota') or self._user_conf.get('mapzen_routing_quota')
+        if routing_quota is '':
+            return 0
+
+        return int(routing_quota)

server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py

@@ -0,0 +1,4 @@
+from geocoder import TomTomGeocoder
+from bulk_geocoder import TomTomBulkGeocoder
+from routing import TomTomRouting, TomTomRoutingResponse
+from isolines import TomTomIsolines, TomTomIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py

@@ -0,0 +1,105 @@
+import json, requests, time
+from requests.adapters import HTTPAdapter
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.geocoder import geocoder_error_response
+from cartodb_services.tomtom import TomTomGeocoder
+from cartodb_services.tools.exceptions import ServiceException
+
+
+class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 1000000  # From the docs
+    MIN_BATCHED_SEARCH = 10  # Batch API is really fast
+    BASE_URL = 'https://api.tomtom.com'
+    BATCH_URL = BASE_URL + '/search/2/batch.json'
+    MAX_STALLED_RETRIES = 100
+    BATCH_RETRY_SLEEP_S = 5
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    def __init__(self, apikey, logger, service_params=None):
+        TomTomGeocoder.__init__(self, apikey, logger, service_params)
+        self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
+        self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
+        self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
+        self.session = requests.Session()
+        self.session.headers.update({'Content-Type': 'application/json'})
+        self.session.mount(self.BATCH_URL,
+                           HTTPAdapter(max_retries=self.max_retries))
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            (search_id, address, city, state, country) = search
+            address = address.encode('utf-8') if address else None
+            city = city.encode('utf-8') if city else None
+            state = state.encode('utf-8') if state else None
+            country = country.encode('utf-8') if country else None
+            result = self.geocode_meta(searchtext=address, city=city,
+                                       state_province=state, country=country)
+            results.append((search_id, result[0], result[1]))
+        return results
+
+    def _batch_geocode(self, searches):
+        full_results = self._geocode_searches(searches)
+        results = []
+        for s, r in zip(searches, full_results):
+            results.append((s[0], r[0], r[1]))
+        return results
+
+    def _geocode_searches(self, searches):
+        try:
+            location = self._send_batch(searches)
+            return self._download_results(location)
+        except Exception as e:
+            msg = "Error running TomTom batch geocode: {}".format(e)
+            self._logger.error(msg, e)
+            return [geocoder_error_response(msg)] * len(searches)
+
+    def _send_batch(self, searches):
+        body = {'batchItems': [{'query': self._query(s)} for s in searches]}
+        request_params = {
+            'key': self._apikey
+        }
+        response = self.session.post(self.BATCH_URL, data=json.dumps(body),
+                                     allow_redirects=False,
+                                     params=request_params,
+                                     timeout=(self.connect_timeout, self.read_timeout))
+        if response.status_code == 303:
+            return response.headers['Location']
+        else:
+            msg = "Error sending batch: {}; Headers: {}".format(
+                response.text.encode('utf-8'), response.headers)
+            self._logger.error(msg)
+            raise ServiceException(msg, response)
+
+    def _download_results(self, location):
+        stalled_retries = 0
+        while True:
+            response = self.session.get(self.BASE_URL + location)
+            if response.status_code == 200:
+                return self._parse_results(response.json())
+            elif response.status_code == 202:
+                stalled_retries += 1
+                if stalled_retries > self.MAX_STALLED_RETRIES:
+                    raise Exception('Too many retries for job {}'.format(location))
+                location = response.headers['Location']
+                time.sleep(self.BATCH_RETRY_SLEEP_S)
+            else:
+                msg = "Error downloading batch: {}; Headers: {}".format(
+                    response.text.encode('utf-8'), response.headers)
+                self._logger.error(msg)
+                raise ServiceException(msg, response)
+
+    def _query(self, search):
+        (search_id, address, city, state, country) = search
+        searchtext = ', '.join(filter(None, [address, city, state]))
+        return self._request_uri(searchtext=searchtext, country=country)
+
+    def _parse_results(self, json_body):
+        return [self._parse_response(item['statusCode'], item['response'])
+                for item in json_body['batchItems']]
+

server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py

@@ -0,0 +1,163 @@
+#!/usr/local/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+import requests
+from uritemplate import URITemplate
+from math import tanh
+from cartodb_services.geocoder import PRECISION_PRECISE, PRECISION_INTERPOLATED, geocoder_metadata, EMPTY_RESPONSE, geocoder_error_response
+from cartodb_services.metrics import Traceable
+from cartodb_services.tools.exceptions import ServiceException
+from cartodb_services.tools.qps import qps_retry
+from cartodb_services.tools.normalize import normalize
+
+HOST = 'https://api.tomtom.com'
+API_BASEURI = '/search/2'
+REQUEST_BASEURI = ('/geocode/'
+               '{searchtext}.json'
+               '?limit=1')
+ENTRY_RESULTS = 'results'
+ENTRY_POSITION = 'position'
+ENTRY_LON = 'lon'
+ENTRY_LAT = 'lat'
+
+SCORE_NORMALIZATION_FACTOR = 0.15
+PRECISION_SCORE_THRESHOLD = 0.5
+MATCH_TYPE_BY_MATCH_LEVEL = {
+    'POI': 'point_of_interest',
+    'Street': 'street',
+    'Address Range': 'street',
+    'Cross Street': 'intersection',
+    'Point Address': 'street_number'
+}
+
+class TomTomGeocoder(Traceable):
+    '''
+    Python wrapper for the TomTom Geocoder service.
+    '''
+
+    def __init__(self, apikey, logger, service_params=None):
+        service_params = service_params or {}
+        self._apikey = apikey
+        self._logger = logger
+
+    def _uri(self, searchtext, country=None):
+        return HOST + API_BASEURI + \
+               self._request_uri(searchtext, country, self._apikey)
+
+    def _request_uri(self, searchtext, country=None, apiKey=None):
+        baseuri = REQUEST_BASEURI
+        if country:
+            baseuri += '&countrySet={}'.format(country)
+        baseuri = baseuri + '&key={apiKey}' if apiKey else baseuri
+        return URITemplate(baseuri).expand(apiKey=apiKey,
+                                           searchtext=searchtext.encode('utf-8'))
+
+    def _extract_lng_lat_from_feature(self, result):
+        position = result[ENTRY_POSITION]
+        longitude = position[ENTRY_LON]
+        latitude = position[ENTRY_LAT]
+        return [longitude, latitude]
+
+    def _validate_input(self, searchtext, city=None, state_province=None,
+                        country=None):
+        if searchtext and searchtext.strip():
+            return True
+        elif city:
+            return True
+        elif state_province:
+            return True
+
+        return False
+
+    @qps_retry(qps=5)
+    def geocode(self, searchtext, city=None, state_province=None,
+                country=None):
+        response = self.geocode_meta(searchtext, city, state_province, country)
+        error_message = response[1].get('error', None)
+        if error_message:
+            raise ServiceException(error_message, None)
+        else:
+            return response[0]
+
+    @qps_retry(qps=5)
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                country=None):
+        if searchtext:
+            searchtext = searchtext.decode('utf-8')
+        if city:
+            city = city.decode('utf-8')
+        if state_province:
+            state_province = state_province.decode('utf-8')
+        if country:
+            country = country.decode('utf-8')
+
+        if not self._validate_input(searchtext, city, state_province, country):
+            return EMPTY_RESPONSE
+
+        address = []
+        if searchtext and searchtext.strip():
+            address.append(normalize(searchtext))
+        if city:
+            address.append(normalize(city))
+        if state_province:
+            address.append(normalize(state_province))
+
+        uri = self._uri(searchtext=', '.join(address), country=country)
+
+        try:
+            response = requests.get(uri)
+            return self._parse_response(response.status_code, response.text)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            msg = 'Timeout connecting to TomTom geocoding server'
+            self._logger.error(msg, te)
+            return geocoder_error_response(msg)
+        except requests.ConnectionError as ce:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to TomTom geocoding server',
+                               exception=ce)
+            return EMPTY_RESPONSE
+
+    def _parse_response(self, status_code, text):
+        if status_code == requests.codes.ok:
+            return self._parse_geocoder_response(text)
+        elif status_code == requests.codes.bad_request:
+            return EMPTY_RESPONSE
+        elif status_code == requests.codes.unprocessable_entity:
+            return EMPTY_RESPONSE
+        else:
+            msg = 'Unknown response {}: {}'.format(str(status_code), text)
+            self._logger.warning('Error parsing TomTom geocoding response',
+                                 data={'msg': msg})
+            return geocoder_error_response(msg)
+
+    def _parse_geocoder_response(self, response):
+        json_response = json.loads(response) \
+            if type(response) != dict else response
+
+        if json_response and json_response[ENTRY_RESULTS]:
+            result = json_response[ENTRY_RESULTS][0]
+            return [
+                self._extract_lng_lat_from_feature(result),
+                self._extract_metadata_from_result(result)
+            ]
+        else:
+            return EMPTY_RESPONSE
+
+    def _extract_metadata_from_result(self, result):
+        score = self._normalize_score(result['score'])
+        match_type = MATCH_TYPE_BY_MATCH_LEVEL.get(result['type'], None)
+        return geocoder_metadata(
+            score,
+            self._precision_from_score(score),
+            [match_type] if match_type else []
+        )
+
+    def _normalize_score(self, score):
+        return tanh(score * SCORE_NORMALIZATION_FACTOR)
+
+    def _precision_from_score(self, score):
+        return PRECISION_PRECISE \
+            if score > PRECISION_SCORE_THRESHOLD else PRECISION_INTERPOLATED

server/lib/python/cartodb_services/cartodb_services/tomtom/isolines.py

@@ -0,0 +1,140 @@
+'''
+Python implementation for TomTom services based isolines.
+'''
+
+import json
+import requests
+from uritemplate import URITemplate
+
+from cartodb_services.tools.exceptions import ServiceException
+from cartodb_services.tools.qps import qps_retry
+from cartodb_services.tools import Coordinate
+from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT,
+                   MAX_SPEEDS)
+
+BASEURI = ('https://api.tomtom.com/routing/1/calculateReachableRange/'
+           '{origin}'
+           '/json'
+           '?key={apikey}'
+           '&timeBudgetInSec={time}'
+           '&travelMode={travelmode}'
+           '&departAt={departat}')
+
+ENTRY_REACHABLERANGE = 'reachableRange'
+ENTRY_BOUNDARY = 'boundary'
+ENTRY_LATITUDE = 'latitude'
+ENTRY_LONGITUDE = 'longitude'
+
+
+class TomTomIsolines():
+    '''
+    Python wrapper for TomTom services based isolines.
+    '''
+
+    def __init__(self, apikey, logger, service_params=None):
+        service_params = service_params or {}
+        self._apikey = apikey
+        self._logger = logger
+
+    def _uri(self, origin, time_range, profile=DEFAULT_PROFILE,
+             date_time=DEFAULT_DEPARTAT):
+        uri = URITemplate(BASEURI).expand(apikey=self._apikey,
+                                          origin=origin,
+                                          time=time_range,
+                                          travelmode=profile,
+                                          departat=date_time)
+        return uri
+
+    def _validate_profile(self, profile):
+        if profile not in VALID_PROFILES:
+            raise ValueError('{profile} is not a valid profile. '
+                             'Valid profiles are: {valid_profiles}'.format(
+                                 profile=profile,
+                                 valid_profiles=', '.join(
+                                     [x for x in VALID_PROFILES])))
+
+    def _parse_coordinates(self, boundary):
+        return [Coordinate(c[ENTRY_LONGITUDE], c[ENTRY_LATITUDE]) for c in boundary]
+
+    def _parse_reachablerange_response(self, response):
+        json_response = json.loads(response)
+
+        if json_response:
+            reachable_range = json_response[ENTRY_REACHABLERANGE]
+
+            return self._parse_coordinates(reachable_range[ENTRY_BOUNDARY])
+
+    @qps_retry(qps=5)
+    def _calculate_isoline(self, origin, time_range,
+                           profile=DEFAULT_PROFILE,
+                           date_time=DEFAULT_DEPARTAT):
+        origin = '{lat},{lon}'.format(lat=origin.latitude,
+                                      lon=origin.longitude)
+
+        uri = self._uri(origin, time_range, profile, date_time)
+        try:
+            response = requests.get(uri)
+
+            if response.status_code == requests.codes.ok:
+                return self._parse_reachablerange_response(response.text)
+            elif response.status_code == requests.codes.bad_request:
+                return []
+            elif response.status_code == requests.codes.unprocessable_entity:
+                return []
+            else:
+                raise ServiceException(response.status_code, response)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            self._logger.error('Timeout connecting to TomTom calculateReachableRange service',
+                               te)
+            raise ServiceException('Error getting calculateReachableRange data from TomTom',
+                                   None)
+        except requests.ConnectionError as ce:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to TomTom calculateReachableRange service',
+                               exception=ce)
+            return []
+
+    def calculate_isochrone(self, origin, time_ranges,
+                            profile=DEFAULT_PROFILE,
+                            date_time=DEFAULT_DEPARTAT):
+        self._validate_profile(profile)
+
+        isochrones = []
+        for time_range in time_ranges:
+            coordinates = self._calculate_isoline(origin=origin,
+                                                  time_range=time_range,
+                                                  profile=profile,
+                                                  date_time=date_time)
+            isochrones.append(TomTomIsochronesResponse(coordinates,
+                                                       time_range))
+        return isochrones
+
+    def calculate_isodistance(self, origin, distance_range,
+                              profile=DEFAULT_PROFILE,
+                              date_time=DEFAULT_DEPARTAT):
+        self._validate_profile(profile)
+
+        max_speed = MAX_SPEEDS[profile]
+        time_range = distance_range / max_speed
+
+        return self._calculate_isoline(origin=origin,
+                                       time_range=time_range,
+                                       profile=profile,
+                                       date_time=date_time)
+
+
+class TomTomIsochronesResponse:
+
+    def __init__(self, coordinates, duration):
+        self._coordinates = coordinates
+        self._duration = duration
+
+    @property
+    def coordinates(self):
+        return self._coordinates
+
+    @property
+    def duration(self):
+        return self._duration

server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py

@@ -0,0 +1,144 @@
+'''
+Python client for the TomTom Routing service.
+'''
+
+import json
+import requests
+from uritemplate import URITemplate
+from cartodb_services.metrics import Traceable
+from cartodb_services.tools import PolyLine
+from cartodb_services.tools.coordinates import (validate_coordinates,
+                                                marshall_coordinates)
+from cartodb_services.tools.exceptions import ServiceException
+from cartodb_services.tools.qps import qps_retry
+from types import (DEFAULT_PROFILE, VALID_PROFILES, DEFAULT_DEPARTAT)
+
+BASEURI = ('https://api.tomtom.com/routing/1/calculateRoute/'
+           '{coordinates}'
+           '/json'
+           '?key={apikey}'
+           '&travelMode={travelmode}'
+           '&departAt={departat}'
+           '&computeBestOrder=true')
+
+NUM_WAYPOINTS_MIN = 2
+NUM_WAYPOINTS_MAX = 20
+
+ENTRY_ROUTES = 'routes'
+ENTRY_SUMMARY = 'summary'
+ENTRY_LENGTH = 'lengthInMeters'
+ENTRY_TIME = 'travelTimeInSeconds'
+ENTRY_LEGS = 'legs'
+ENTRY_POINTS = 'points'
+ENTRY_LATITUDE = 'latitude'
+ENTRY_LONGITUDE = 'longitude'
+
+
+class TomTomRouting(Traceable):
+    '''
+    Python wrapper for the TomTom Routing service.
+    '''
+
+    def __init__(self, apikey, logger, service_params=None):
+        service_params = service_params or {}
+        self._apikey = apikey
+        self._logger = logger
+
+    def _uri(self, coordinates, profile=DEFAULT_PROFILE,
+             date_time=DEFAULT_DEPARTAT):
+        uri = URITemplate(BASEURI).expand(apikey=self._apikey,
+                                          coordinates=coordinates,
+                                          travelmode=profile,
+                                          departat=date_time)
+        return uri
+
+    def _validate_profile(self, profile):
+        if profile not in VALID_PROFILES:
+            raise ValueError('{profile} is not a valid profile. '
+                             'Valid profiles are: {valid_profiles}'.format(
+                                 profile=profile,
+                                 valid_profiles=', '.join(
+                                     [x for x in VALID_PROFILES])))
+
+    def _marshall_coordinates(self, coordinates):
+        return ':'.join(['{lat},{lon}'.format(lat=coordinate.latitude,
+                                              lon=coordinate.longitude)
+                         for coordinate in coordinates])
+
+    def _parse_routing_response(self, response):
+        json_response = json.loads(response)
+
+        if json_response:
+            route = json_response[ENTRY_ROUTES][0]  # Force the first route
+
+            geometry = self._parse_legs(route[ENTRY_LEGS])
+            summary = route[ENTRY_SUMMARY]
+            distance = summary[ENTRY_LENGTH]
+            duration = summary[ENTRY_TIME]
+
+            return TomTomRoutingResponse(geometry, distance, duration)
+        else:
+            return TomTomRoutingResponse(None, None, None)
+
+    def _parse_legs(self, legs):
+        geometry = []
+        for leg in legs:
+            points = leg[ENTRY_POINTS]
+            for point in points:
+                geometry.append((point[ENTRY_LATITUDE],
+                                 point[ENTRY_LONGITUDE]))
+        return geometry
+
+    @qps_retry(qps=5)
+    def directions(self, waypoints, profile=DEFAULT_PROFILE,
+                   date_time=DEFAULT_DEPARTAT):
+        self._validate_profile(profile)
+        validate_coordinates(waypoints, NUM_WAYPOINTS_MIN, NUM_WAYPOINTS_MAX)
+
+        coordinates = self._marshall_coordinates(waypoints)
+
+        uri = self._uri(coordinates, profile, date_time)
+
+        try:
+            response = requests.get(uri)
+
+            if response.status_code == requests.codes.ok:
+                return self._parse_routing_response(response.text)
+            elif response.status_code == requests.codes.bad_request:
+                return TomTomRoutingResponse(None, None, None)
+            elif response.status_code == requests.codes.unprocessable_entity:
+                return TomTomRoutingResponse(None, None, None)
+            else:
+                raise ServiceException(response.status_code, response)
+        except requests.Timeout as te:
+            # In case of timeout we want to stop the job because the server
+            # could be down
+            self._logger.error('Timeout connecting to TomTom routing service',
+                               te)
+            raise ServiceException('Error getting routing data from TomTom',
+                                   None)
+        except requests.ConnectionError as ce:
+            # Don't raise the exception to continue with the geocoding job
+            self._logger.error('Error connecting to TomTom routing service',
+                               exception=ce)
+            return TomTomRoutingResponse(None, None, None)
+
+
+class TomTomRoutingResponse:
+
+    def __init__(self, shape, length, duration):
+        self._shape = shape
+        self._length = length
+        self._duration = duration
+
+    @property
+    def shape(self):
+        return self._shape
+
+    @property
+    def length(self):
+        return self._length
+
+    @property
+    def duration(self):
+        return self._duration

server/lib/python/cartodb_services/cartodb_services/tomtom/types.py

@@ -0,0 +1,26 @@
+TOMTOM_ROUTING_APIKEY_ROUNDROBIN = 'tomtom_routing_apikey_roundrobin'
+TOMTOM_GEOCODER_APIKEY_ROUNDROBIN = 'tomtom_geocoder_apikey_roundrobin'
+TOMTOM_ISOLINES_APIKEY_ROUNDROBIN = 'tomtom_isolines_apikey_roundrobin'
+
+PROFILE_DRIVING = 'car'
+PROFILE_CYCLING = 'bicycle'
+PROFILE_WALKING = 'pedestrian'
+DEFAULT_PROFILE = PROFILE_DRIVING
+
+DEFAULT_DEPARTAT = 'now'
+
+VALID_PROFILES = [PROFILE_DRIVING,
+                  PROFILE_CYCLING,
+                  PROFILE_WALKING]
+
+MAX_SPEEDS = {
+    PROFILE_WALKING: 3.3333333,  # In m/s, assuming 12km/h walking speed
+    PROFILE_CYCLING: 16.67,  # In m/s, assuming 60km/h max speed
+    PROFILE_DRIVING: 41.67  # In m/s, assuming 140km/h max speed
+}
+
+TRANSPORT_MODE_TO_TOMTOM = {
+    'car': 'car',
+    'walk': 'pedestrian',
+    'bicycle': 'bicycle',
+}

server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py

@@ -55,7 +55,7 @@ def coordinates_to_polygon(coordinates):
     wkt_coordinates = ','.join(result_coordinates)
 
     try:
-        sql = "SELECT ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3) as geom".format(wkt_coordinates)
+        sql = "SELECT st_multi(ST_CollectionExtract(ST_MakeValid(ST_MakePolygon(ST_GeomFromText('LINESTRING({0})', 4326))),3)) as geom".format(wkt_coordinates)
         geometry = plpy.execute(sql, 1)[0]['geom']
     except BaseException as e:
         plpy.warning("Can't generate POLYGON from coordinates: {0}".format(e))

server/lib/python/cartodb_services/cartodb_services/tools/log.py

@@ -35,28 +35,28 @@ class Logger:
             return
         self._send_to_rollbar('debug', text, exception, data)
         self._send_to_log_file('debug', text, exception, data)
-        self._send_to_plpy('debug', text)
+        self._send_to_plpy('debug', text, exception)
 
     def info(self, text, exception=None, data={}):
         if not self._check_min_level('info'):
             return
         self._send_to_rollbar('info', text, exception, data)
         self._send_to_log_file('info', text, exception, data)
-        self._send_to_plpy('info', text)
+        self._send_to_plpy('info', text, exception)
 
     def warning(self, text, exception=None, data={}):
         if not self._check_min_level('warning'):
             return
         self._send_to_rollbar('warning', text, exception, data)
         self._send_to_log_file('warning', text, exception, data)
-        self._send_to_plpy('warning', text)
+        self._send_to_plpy('warning', text, exception)
 
     def error(self, text, exception=None, data={}):
         if not self._check_min_level('error'):
             return
         self._send_to_rollbar('error', text, exception, data)
         self._send_to_log_file('error', text, exception, data)
-        self._send_to_plpy('error', text)
+        self._send_to_plpy('error', text, exception)
 
     def _check_min_level(self, level):
         return True if self.LEVELS[level] >= self._min_level else False
@@ -85,18 +85,31 @@ class Logger:
             elif level == 'error':
                 self._file_logger.error(text, extra=extra_data)
 
-    def _send_to_plpy(self, level, text):
+    def _send_to_plpy(self, level, text, exception=None):
+        # exception might also be a tuple generated by sys.exc_info
+        if exception:
+            if isinstance(exception, tuple) and len(exception) > 1:
+                exception = exception[1]
+            exception_message = '. Exception: {}'.format(exception)
+        else:
+            exception_message = ''
+
+        # Adding trace breaks tests
+        # trace = traceback.format_exc(15)
+        # message = '{}{}. Trace: {}'.format(text, exception_message, trace)
+        message = '{}{}'.format(text, exception_message)
+
         if self._check_plpy():
             if level == 'debug':
-                plpy.debug(text)
+                plpy.debug(message)
             elif level == 'info':
-                plpy.info(text)
+                plpy.info(message)
             elif level == 'warning':
-                plpy.warning(text)
+                plpy.warning(message)
             elif level == 'error':
                 # Plpy.error and fatal raises exceptions and we only want to
                 # log an error, exceptions should be raise explicitly
-                plpy.warning(text)
+                plpy.warning(message)
 
     def _parse_log_extra_data(self, exception, data):
         extra_data = {}

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.16.7',
+    version='0.20.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/credentials.py

@@ -0,0 +1,11 @@
+import os
+
+
+def mapbox_api_key():
+    """Returns Mapbox API key. Requires setting MAPBOX_API_KEY environment variable."""
+    return os.environ['MAPBOX_API_KEY']
+
+
+def tomtom_api_key():
+    """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable."""
+    return os.environ['TOMTOM_API_KEY']

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
 
 class TestGeocoderUserConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -28,6 +28,9 @@ class TestGeocoderUserConfig(TestCase):
             elif geocoder_provider == 'mapbox':
                 assert geocoder_config.mapbox_geocoder is True
                 assert geocoder_config.geocoding_quota == 100
+            elif geocoder_provider == 'tomtom':
+                assert geocoder_config.tomtom_geocoder is True
+                assert geocoder_config.geocoding_quota == 100
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None
@@ -81,7 +84,7 @@ class TestGeocoderUserConfig(TestCase):
 
 class TestGeocoderOrgConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -107,6 +110,9 @@ class TestGeocoderOrgConfig(TestCase):
             elif geocoder_provider == 'mapbox':
                 assert geocoder_config.mapbox_geocoder is True
                 assert geocoder_config.geocoding_quota == 200
+            elif geocoder_provider == 'tomtom':
+                assert geocoder_config.tomtom_geocoder is True
+                assert geocoder_config.geocoding_quota == 200
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None
@@ -167,7 +173,8 @@ class TestGeocoderOrgConfig(TestCase):
 
 
 class TestIsolinesUserConfig(TestCase):
-    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
+    # Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -183,6 +190,8 @@ class TestIsolinesUserConfig(TestCase):
                 assert isolines_config.service_type is 'mapzen_isolines'
             elif isolines_provider is 'mapbox':
                 assert isolines_config.service_type is 'mapbox_isolines'
+            elif isolines_provider is 'tomtom':
+                assert isolines_config.service_type is 'tomtom_isolines'
             else:
                 assert isolines_config.service_type is 'here_isolines'
             assert isolines_config.isolines_quota == 100
@@ -225,8 +234,8 @@ class TestIsolinesUserConfig(TestCase):
 
 
 class TestIsolinesOrgConfig(TestCase):
-
-    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'mapbox']
+    # Don't test mapbox. See CartoDB/cartodb-management/issues/5199"
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -301,7 +310,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
         orgname = None
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 1000
+        assert config.routing_quota == 1000
 
     def test_org_quota_overrides_user_quota(self):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
@@ -315,7 +324,7 @@ class TestRoutingConfig(TestCase):
         self._redis_conn.hset(orgname_key, 'here_isolines_quota', 0)
 
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 5000
+        assert config.routing_quota == 5000
 
     def test_should_have_soft_limit_false_by_default(self):
         orgname = None
@@ -335,41 +344,7 @@ class TestDataObservatoryUserConfig(TestCase):
         self.redis_conn = MockRedis()
         plpy_mock_config()
 
-    def test_should_return_config_for_obs_snapshot(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=100, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.monthly_quota == 100
-        assert do_config.soft_limit is False
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_true_if_soft_limit_is_true_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, soft_limit=True, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.soft_limit is True
-
-    def test_should_return_0_if_quota_is_0_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota=0, end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_0_if_quota_is_empty_in_redis(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
-                                quota='', end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                              'test_user')
-        assert do_config.monthly_quota == 0
-
-    def test_should_return_config_for_obs_snapshot(self):
+    def test_should_return_config_for_obs_config(self):
         yesterday = datetime.today() - timedelta(days=1)
         build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
                                 quota=100, end_date=yesterday)