Merge pull request #341 from CartoDB/development

Release client 0.15.0 and server 0.21.0

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+---
+
+The issue tracker is at [github.com/CartoDB/dataservices-api](https://github.com/CartoDB/dataservices-api).
+
+We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
+
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

NEWS.md

@@ -1,3 +1,34 @@
+February 2st, 2017
+===================
+* Version 0.21.0 of the server and version 0.15.0 of the client
+  * Added functions `OBS_GetData` and `OBS_GetMeta`
+  * Default isolines provider changed to `mapzen` instead of `heremaps`
+
+November 29st, 2016
+===================
+* Version 0.20.0 of the server and version 0.12.0 of the python library
+  * Added integration with the new Mapzen isochrones functionality
+
+November 25st, 2016
+===================
+* Version 0.19.0 of the server, version 0.11.0 of the python library and version 0.13.0 of the client
+  * functions to check the quota, both server and client
+  * removed the no_params from the templates (this caused trouble, not needed anymore)
+  * bug fixes: observatory quota, quotas as integers, mapzen geocoder soft limit as bool
+
+November 21st, 2016
+===================
+* Version 0.18.1 of the server and version 0.12.1 of the client
+  * Add new fields to the obs_meta_geometry due to new changes introduced in the DO 1.1.2 release
+
+November 11st, 2016
+===================
+* Version 0.18.0 of the server and version 0.12.0 of the client
+  * Added obs_legacybuildermetada functions to grab the needed metadata in the builder while making a data enrichment analysis. Closes #286
+  * Added metadata functions that will be used in the future to gather the metadata for the new data enrichment UI. Closes #287
+  * Fixed integration test for street geocoding
+  * Makefile now has a new task to create a new release in the client part, as was in the server, using make release NEW_VERSION=x.x.x
+
 November 7st, 2016
 ==================
 * Version 0.17.0 of the server and version 0.10.0 of the python package

client/.gitignore

@@ -2,9 +2,10 @@ results/
 regression.diffs
 regression.out
 20_public_functions.sql
+25_exception_safe_private_functions.sql
 30_plproxy_functions.sql
 90_grant_execute.sql
 cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
-cdb_geocoder_client--0.3.0.sql
+cdb_geocoder_client--0.3.0.sql

client/Makefile

@@ -57,9 +57,11 @@ all: $(DATA)
 .PHONY: release
 release: $(EXTENSION).control $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
-	mv *.sql old_versions
+	git mv *.sql old_versions
 	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 

client/cdb_dataservices_client--0.14.1--0.15.0.sql

@@ -0,0 +1,204 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.15.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomvals geomval[] ,params json ,merge boolean DEFAULT true)
+RETURNS TABLE(id int, data json) AS $$
+DECLARE
+
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+    RETURN QUERY
+    SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge);
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getdata (geomrefs text[] ,params json)
+RETURNS TABLE(id text, data json) AS $$
+DECLARE
+
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+    RETURN QUERY
+    SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params);
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeta (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL)
+RETURNS json AS $$
+DECLARE
+  ret json;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+    SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret;
+    RETURN ret;
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomvals geomval[] ,params json ,merge boolean DEFAULT true)
+RETURNS TABLE(id int, data json) AS $$
+DECLARE
+
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+    BEGIN
+      RETURN QUERY
+      SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomvals, params, merge);
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+    END;
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata_exception_safe (geomrefs text[] ,params json)
+RETURNS TABLE(id text, data json) AS $$
+DECLARE
+
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+    BEGIN
+      RETURN QUERY
+      SELECT * FROM cdb_dataservices_client._obs_getdata(username, orgname, geomrefs, params);
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+    END;
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe (geom_ref Geometry(Geometry, 4326) ,params json ,max_timespan_rank integer DEFAULT NULL ,max_score_rank integer DEFAULT NULL ,target_geoms integer DEFAULT NULL)
+RETURNS json AS $$
+DECLARE
+  ret json;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+    BEGIN
+      SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, organization_name text, geomvals geomval[], params json, merge boolean DEFAULT true)
+RETURNS TABLE(id int, data json) AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT * FROM cdb_dataservices_server.obs_getdata (username, organization_name, geomvals, params, merge);
+
+$$ LANGUAGE plproxy;
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getdata (username text, organization_name text, geomrefs text[], params json)
+RETURNS TABLE(id text, data json) AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT * FROM cdb_dataservices_server.obs_getdata (username, organization_name, geomrefs, params);
+
+$$ LANGUAGE plproxy;
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeta (username text, organization_name text, geom_ref Geometry(Geometry, 4326), params json, max_timespan_rank integer DEFAULT NULL, max_score_rank integer DEFAULT NULL, target_geoms integer DEFAULT NULL)
+RETURNS json AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT cdb_dataservices_server.obs_getmeta (username, organization_name, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms);
+
+$$ LANGUAGE plproxy;

client/cdb_dataservices_client--0.15.0--0.14.1.sql

@@ -0,0 +1,17 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.1'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (text, text, geomval[], json, boolean);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (text, text, text[], json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (text, text, Geometry(Geometry, 4326), json, integer, integer, integer);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata_exception_safe (geomval[], json, boolean);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata_exception_safe (text[], json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta_exception_safe (Geometry(Geometry, 4326), json , integer, integer, integer);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdata (geomval[], json, boolean);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdata (text[], json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getmeta (Geometry(Geometry, 4326), json, integer, integer, integer);

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.13.0'
+default_version = '0.15.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.14.0--0.13.0.sql

@@ -0,0 +1,52 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.13.0'" to load this file. \quit
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC);
+

client/old_versions/cdb_dataservices_client--0.14.0--0.14.1.sql

@@ -0,0 +1,9 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.1'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- This release introduces no changes other than the use of
+-- search path in the install and migration scripts

client/old_versions/cdb_dataservices_client--0.14.1--0.14.0.sql

@@ -0,0 +1,9 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- This release introduces no changes other than the use of
+-- search path in the install and migration scripts

client/renderer/interface.yaml

@@ -261,6 +261,38 @@
     - { name: boundary_id, type: text}
     - { name: time_span, type: text, default: 'NULL'}
 
+- name: obs_getdata
+  return_type: TABLE(id int, data json)
+  multi_row: true
+  multi_field: true
+  table_fields:
+    - { name: id, type: integer }
+    - { name: data, type: json }
+  params:
+    - { name: geomvals,  type: "geomval[]" }
+    - { name: params, type: json }
+    - { name: merge, type: boolean, default: true }
+
+- name: obs_getdata
+  return_type: TABLE(id text, data json)
+  multi_row: true
+  multi_field: true
+  table_fields:
+    - { name: id, type: text }
+    - { name: data, type: json }
+  params:
+    - { name: geomrefs,  type: "text[]" }
+    - { name: params, type: json }
+
+- name: obs_getmeta
+  return_type: json
+  params:
+    - { name: geom_ref,  type: "Geometry(Geometry, 4326)" }
+    - { name: params, type: json }
+    - { name: max_timespan_rank, type: integer, default: 'NULL' }
+    - { name: max_score_rank, type: integer, default: 'NULL' }
+    - { name: target_geoms, type: integer, default: 'NULL' }
+
 - name: obs_getcategory
   return_type: text
   params:
@@ -388,4 +420,4 @@
   return_type: BOOLEAN
   params:
     - { name: service, type: TEXT }
-    - { name: input_size, type: NUMERIC }
+    - { name: input_size, type: NUMERIC }

client/renderer/templates/25_exception_safe_private_functions.erb

@@ -0,0 +1,61 @@
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>)
+RETURNS <%= return_type %> AS $$
+DECLARE
+  <% if not multi_row %>ret <%= return_type %>;<% end %>
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  <% if multi_row %>
+    BEGIN
+      RETURN QUERY
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+    END;
+  <% elsif multi_field %>
+    BEGIN
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% else %>
+    BEGIN
+      SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% end %>
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

client/renderer/templates/90_grant_execute.erb

@@ -1 +1,2 @@
 GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;
+GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser;

client/sql/00_header.sql

@@ -1,3 +1,6 @@
 --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
 -- Complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;

client/test/expected/25_exception_safe_private_functions_test.out

@@ -0,0 +1,56 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+-- Use regular user role
+SET ROLE test_regular_user;
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+WARNING:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
+PL/pgSQL function _cdb_geocode_street_point_exception_safe(text,text,text,text) line 21 at SQL statement
+ _cdb_geocode_street_point_exception_safe 
+------------------------------------------
+ 
+(1 row)
+
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+WARNING:  cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  PL/pgSQL function _cdb_isodistance_exception_safe(geometry,text,integer[],text[]) line 21 at RETURN QUERY
+ center | data_range | the_geom 
+--------+------------+----------
+(0 rows)
+
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+WARNING:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)"
+PL/pgSQL function _cdb_route_point_to_point_exception_safe(geometry,geometry,text,text[],text) line 21 at SQL statement
+ shape | length | duration 
+-------+--------+----------
+       |        |         
+(1 row)
+

client/test/expected/90_data_observatory_test.out

@@ -173,6 +173,27 @@ BEGIN
   RETURN QUERY SELECT '0106000020E61000000100000001030000000100000019000000DD96C805677C52C0CBD8D0CDFE5A44407A19C5724B7C52C033BF9A03045B4440FF59F3E32F7C52C0CB290131095B4440247EC51A2E7C52C0548B8862F25A44401FF5D72B2C7C52C0CB811E6ADB5A444049F086342A7C52C09CC3B5DAC35A444044679945287C52C01F680586AC5A444012D90759167C52C09430D3F6AF5A44405698BED7107C52C04759BF99985A4440DEE522BE137C52C00664AF777F5A4440F92EA52E197C52C09BA73AE4665A4440C558A65F227C52C0D80DDB16655A4440373465A71F7C52C065A71FD4455A44404AED45B41D7C52C0785DBF60375A444083A3E4D5397C52C09946938B315A4440F2B4FCC0557C52C01FF5D72B2C5A444026C286A7577C52C02BDD5D67435A44405BCF108E597C52C0C651B9895A5A444030D461855B7C52C0D1393FC5715A444065E1EB6B5D7C52C078280AF4895A44403AE63C635F7C52C07D1F0E12A25A44403F6F2A52617C52C05983F755B95A44405C3AE63C637C52C04852D2C3D05A444049810530657C52C0FAEE5696E85A4440DD96C805677C52C0CBD8D0CDFE5A4440'::geometry as the_geom, '36047049300'::text as geom_refs;
 END;
 $$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms;
+  RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json)
+RETURNS TABLE (id TEXT, data JSON) AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params;
+  RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true)
+RETURNS TABLE (id TEXT, data JSON) AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge;
+  RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
+END;
+$$ LANGUAGE 'plpgsql';
 -- Exercise the public and the proxied function
 SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text);
 NOTICE:  cdb_dataservices_client._obs_get_demographic_snapshot(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_get_demographic_snapshot invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, 2009 - 2013, "us.census.tiger".block_group)
@@ -337,3 +358,34 @@ CONTEXT:  PL/pgSQL function obs_getavailableboundaries(geometry,text) line 16 at
  (us.census.tiger.place,"Incorporated places are those reported to the Census Bureau as legally in existence as of January 1, 2010, as reported in the latest Boundary and Annexation Survey (BAS), under the laws of their respective states.",2014,obs_7c9493c41fa8f4bd178ab993ea3d5891c1977667)
 (1 row)
 
+SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000);
+NOTICE:  cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], 1, 1, 1000)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
+PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
+                                                                                                                                                                                                                                                                                                                                                                                                                                         obs_getmeta                                                                                                                                                                                                                                                                                                                                                                                                                                          
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]
+(1 row)
+
+SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000));
+NOTICE:  cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}], 1, 1, 1000)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
+PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
+NOTICE:  cdb_dataservices_client._obs_getdata(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, <NULL>, {36047}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}])
+CONTEXT:  PL/pgSQL function obs_getdata(text[],json) line 16 at RETURN QUERY
+                obs_getdata                 
+--------------------------------------------
+ (36047,"[{""value"" : 10349.1547875017}]")
+(1 row)
+
+SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]'));
+NOTICE:  cdb_dataservices_client._obs_getmeta(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeta invoked with params (test_user, <NULL>, 0101000020E61000009A999999997952C09A99999999594440, [{"numer_id": "us.census.acs.B01003001"}], <NULL>, <NULL>, <NULL>)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_getmeta(username, orgname, geom_ref, params, max_timespan_rank, max_score_rank, target_geoms)"
+PL/pgSQL function obs_getmeta(geometry,json,integer,integer,integer) line 16 at SQL statement
+NOTICE:  cdb_dataservices_client._obs_getdata(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getdata invoked with params (test_user, <NULL>, {"(0101000020E61000009A999999997952C09A99999999594440,1)"}, [{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}], t)
+CONTEXT:  PL/pgSQL function obs_getdata(geomval[],json,boolean) line 16 at RETURN QUERY
+                obs_getdata                 
+--------------------------------------------
+ (36047,"[{""value"" : 10349.1547875017}]")
+(1 row)
+

client/test/sql/25_exception_safe_private_functions_test.sql

@@ -0,0 +1,41 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+
+-- Use regular user role
+SET ROLE test_regular_user;
+
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);

client/test/sql/90_data_observatory_test.sql

@@ -194,6 +194,30 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeta(username text, orgname text, geom geometry(Geometry, 4326), params JSON, max_timespan_rank INTEGER DEFAULT NULL, max_score_rank INTEGER DEFAULT NULL, target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getmeta invoked with params (%, %, %, %, %, %, %)', username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms;
+  RETURN '[{"id" : 1, "numer_id" : "us.census.acs.B01003001", "timespan_rank" : 1, "score_rank" : 1, "score" : 19.9580760018781868330120152747832081562684, "numer_aggregate" : "sum", "numer_colname" : "total_pop", "numer_geomref_colname" : "geoid", "numer_tablename" : "obs_209d3476ef8eaaa18e597cabcf1bdb627f37aa5e", "numer_type" : "Numeric", "denom_aggregate" : null, "denom_colname" : null, "denom_geomref_colname" : null, "denom_tablename" : null, "denom_type" : null, "geom_colname" : "the_geom", "geom_geomref_colname" : "geoid", "geom_tablename" : "obs_78fb6c1d6ff6505225175922c2c389ce48d7632c", "geom_type" : "Geometry", "geom_timespan" : "2015", "numer_timespan" : "2011 - 2015", "numer_name" : "Total Population", "denom_name" : null, "geom_name" : "US Census Block Groups", "normalization" : null, "denom_id" : null, "geom_id" : "us.census.tiger.block_group"}]'::JSON;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomrefs text[], params json)
+RETURNS TABLE (id TEXT, data JSON) AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %)', username, orgname, geomrefs, params;
+  RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getdata(username text, orgname text, geomvals geomval[], params json, merge boolean default true)
+RETURNS TABLE (id TEXT, data JSON) AS $$
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getdata invoked with params (%, %, %, %, %)', username, orgname, geomvals, params, merge;
+  RETURN QUERY SELECT '36047'::TEXT AS id, '[{"value" : 10349.1547875017}]'::JSON AS data;
+END;
+$$ LANGUAGE 'plpgsql';
+
 -- Exercise the public and the proxied function
 SELECT obs_get_demographic_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '2009 - 2013'::text, '"us.census.tiger".block_group'::text);
 SELECT obs_get_segment_snapshot(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), '"us.census.tiger".block_group'::text);
@@ -214,3 +238,7 @@ SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326
 SELECT obs_getpopulation(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326));
 SELECT obs_search('total_pop'::text);
 SELECT obs_getavailableboundaries(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326));
+SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]', 1, 1, 1000);
+SELECT obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001", "geom_id": "us.census.tiger.county"}]', 1, 1, 1000));
+SELECT obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), '[{"numer_id": "us.census.acs.B01003001"}]'));
+

client/upgrade_downgrade_template.erb

@@ -2,4 +2,7 @@
 -- Complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
 
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
 -- HERE goes your code to upgrade/downgrade

doc/internal/exception_safe.md

@@ -0,0 +1,36 @@
+# Exception-Safe functions
+
+The public API dataservices functions emit exceptions in general when an error occurs
+or a limiting condition is met (e.g. quotas are exceeded).
+
+For each public function `func` we have a internal function named `_func_exception_safe` which
+acts as a wrapper to the public function, with the same signature, but captures
+exceptions generated during its execution (except those due to incomplete configuration or
+authentication issues) and returns NULL or empty set values in those cases.
+
+Please note these functions are considered **not public** and therefore their API (including which exceptions are wrapped and which ones are not) may change.
+
+Instead of raising an exception they raise warnings, hopefully containing the same information of the original exception.
+
+## Intended Use
+
+These functions are useful in cases when it is undesirable to rollback a transaction.
+Fo example if a table is geocoded with:
+
+```sql
+UPDATE table SET the_geom=cdb_geocode_street_point(user,NULL,address,city,NULL,country);
+```
+
+In case of the user geocoding quota being exhausted mid-process, the user could
+incur in external service expenses but any geocoded data would be lost due to the
+transaction rollback.
+
+We can avoid the problem using the corresponding exception-safe function:
+
+```sql
+UPDATE table SET the_geom=_cdb_geocode_street_point_exception_safe(user,NULL,address,city,NULL,country);
+```
+
+# Addition Information
+
+See https://github.com/CartoDB/dataservices-api/issues/314 for more information.

doc/quota_information.md

@@ -12,3 +12,124 @@ Quota consumption is calculated based on the number of request made for each fun
   **Note:** Queries to the Data Services API, and any of its functions in your maps, may be forbidden in the future
 
 * It is advised to store results of these queries into your datasets, and refresh them as needed. This ensure more control of quota credits for your account
+
+
+## Quota Information Functions
+
+There are several SQL functions that you can run to obtain quota information about your services. 
+
+## cdb_service_quota_info()
+
+Returns information about per-service quotas (available and used) for the account.
+
+#### Returns
+
+This function returns a set of service quota information records, one per service.
+
+Name            | Type      | Description
+--------------- | --------- | ------------
+`service`       | `text`    | Type of service.
+`monthly_quota` | `numeric` | Quota available to the user (number of calls) per monthly period.
+`used_quota`    | `numeric` | Quota used by the user in the present period.
+`soft_limit`    | `boolean` | Set to `True`, if the user has *soft-limit* quota.
+`provider`      | `text`    | Service provider for this type of service.
+
+Service Types:
+
+* `'isolines'` [Isoline/Isochrones (isochrone/isodistance lines) service](https://carto.com/docs/carto-engine/dataservices-api/isoline_functions/)
+* `'hires_geocoder'` [Street level geocoding](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions#street-level-geocoder)
+* `'routing'` [Routing functions](https://carto.com/docs/carto-engine/dataservices-api/routing_functions/)
+* `'observatory'` Data Observatory services ([demographic](https://carto.com/docs/carto-engine/dataservices-api/demographic_functions/) and [segmentation](https://carto.com/docs/carto-engine/dataservices-api/segmentation_functions/) functions)
+
+**Notes**
+
+Users who have *soft-quota* activated never run out of quota, but they may incur extra
+expenses when the regular quota is exceeded.
+
+A zero value of `monthly_quota` indicates that the service has not been activated for the user.
+
+#### Example
+
+```sql
+SELECT * FROM cdb_service_quota_info();
+```
+
+Result:
+
+```sql
+    service     | monthly_quota | used_quota | soft_limit |     provider
+----------------+---------------+------------+------------+------------------
+ isolines       |           100 |          0 | f          | mapzen
+ hires_geocoder |           100 |          0 | f          | mapzen
+ routing        |            50 |          0 | f          | mapzen
+ observatory    |             0 |          0 | f          | data observatory
+(4 rows)
+
+```
+
+In this case, notice that the user has no access to the observatory services. All quotas are *hard-limited* (no soft limits), and no quota has been used in the present period.
+
+## cdb_enough_quota(service text ,input_size numeric)
+
+This function is useful to check if enough quota is available for completing a job.
+
+This is specifically relevant if a number of service calls are to be performed inside a transaction. If any of the calls fails (due to exceeded quota), the transaction will be rolled back; resulting in partial quota consumption, but no saved results from the services consumed.
+
+**Tip:** If you are requesting repeating quota-consuming functions (e.g. to geocode a whole table), it is extremely important to check if enough quota is available to complete the job _before_ applying this function.
+
+Note that some services consume more than one credit per row/call. For example, isolines (with more than one range/track) consume (N rows x M ranges) credits; indicating that the input size should be N x M.
+
+#### Arguments
+
+Name         | Type      | Description
+------------ | --------- | -----------
+`service`    | `text`    | Service to check; see the list of valid services above.
+`input_size` | `numeric` | Number of service calls required, i.e. size of the input to be processed.
+
+#### Returns
+
+The result is a *boolean* value. A *true* value (`'t'`) indicates that the available quota
+for the service is enough for the input size requested. A *false* value (`'f'`) indicates
+insufficient quota.
+
+#### Example
+
+Suppose you want to geocode a whole table. In order to check that you have enough quota, and avoid a "quota exhausted" exception, first find out how many records you need to geocode:
+
+```sql
+SELECT COUNT(*) FROM {tablename} WHERE {street_name_column} IS NOT NULL;
+```
+
+Result: here's a sample result of 10000 records:
+
+```sql
+ count
+-------
+  10000
+(1 row)
+```
+
+The result shows how much quota is needed to complete this job. In this case,
+each call to `cdb_geocode_street_point` consumes one quota credit. This indicates that we need one credit per row to geocode the whole table.
+
+```sql
+SELECT cdb_enough_quota('hires_geocoder', {number_of_records});
+```
+
+The result is similar to the following:
+
+```sql
+ cdb_enough_quota
+------------------
+ t
+```
+
+If the result of this query is *true* (`'t'`), you can safely proceed. If a *false* value (`'f'`) is returned, you should avoid processing any more requests that consume quota. Apply the `cdb_service_quota_info` function to get more information about your services.
+
+**Note:** Remember to apply any filtering conditions that you used to count the records (in this case, `{street_name_column} IS NOT NULL`):
+
+
+```sql
+UPDATE {tablename} SET the_geom = cdb_geocode_street_point({street_name_column})
+  WHERE {street_name_column} IS NOT NULL;
+```

server/extension/cdb_dataservices_server--0.20.0--0.21.0.sql

@@ -0,0 +1,174 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.21.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomvals geomval[],
+  params JSON,
+  merge BOOLEAN DEFAULT True)
+RETURNS TABLE (
+  id INT,
+  data JSON
+) AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomvals geomval[],
+  params JSON,
+  merge BOOLEAN DEFAULT True)
+RETURNS TABLE (
+  id INT,
+  data JSON
+) AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdata', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
+        result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
+        empty_results = len(geomvals) - len(result)
+        if empty_results > 0:
+          quota_service.increment_empty_service_use(empty_results)
+        if result:
+          quota_service.increment_success_service_use(len(result))
+          return result
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use(len(geomvals))
+        logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetData')
+    finally:
+        quota_service.increment_total_service_use(len(geomvals))
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomrefs TEXT[],
+  params JSON)
+RETURNS TABLE (
+  id TEXT,
+  data JSON
+) AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomrefs TEXT[],
+  params JSON)
+RETURNS TABLE (
+  id TEXT,
+  data JSON
+) AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdata', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
+        result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
+        empty_results = len(geomrefs) - len(result)
+        if empty_results > 0:
+          quota_service.increment_empty_service_use(empty_results)
+        if result:
+          quota_service.increment_success_service_use(len(result))
+          return result
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use(len(geomrefs))
+        exc_info = sys.exc_info()
+        logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
+        logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetData')
+    finally:
+        quota_service.increment_total_service_use(len(geomrefs))
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
+  username TEXT,
+  orgname TEXT,
+  geom Geometry(Geometry, 4326),
+  params JSON,
+  max_timespan_rank INTEGER DEFAULT NULL,
+  max_score_rank INTEGER DEFAULT NULL,
+  target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
+  username TEXT,
+  orgname TEXT,
+  geom Geometry(Geometry, 4326),
+  params JSON,
+  max_timespan_rank INTEGER DEFAULT NULL,
+  max_score_rank INTEGER DEFAULT NULL,
+  target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  with metrics('obs_getmeta', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
+        if result:
+          return result[0]['meta']
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetMeta')
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.21.0--0.20.0.sql

@@ -0,0 +1,11 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.20.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetData(TEXT, TEXT, geomval[], JSON, BOOLEAN);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetData(TEXT, TEXT, geomval[], JSON, BOOLEAN);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetData(TEXT, TEXT, TEXT[], JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetData(TEXT, TEXT, TEXT[], JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetMeta(TEXT, TEXT, Geometry(Geometry, 4326), JSON, INTEGER, INTEGER, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetMeta(TEXT, TEXT, Geometry(Geometry, 4326), JSON, INTEGER, INTEGER, INTEGER);

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.20.0'
+default_version = '0.21.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/110_data_observatory_augmentation.sql

@@ -567,3 +567,173 @@ RETURNS NUMERIC AS $$
     finally:
         quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomvals geomval[],
+  params JSON,
+  merge BOOLEAN DEFAULT True)
+RETURNS TABLE (
+  id INT,
+  data JSON
+) AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetData(geomvals, params, merge);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomvals geomval[],
+  params JSON,
+  merge BOOLEAN DEFAULT True)
+RETURNS TABLE (
+  id INT,
+  data JSON
+) AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdata', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4, $5);", ["text", "text", "geomval[]", "json", "boolean"])
+        result = plpy.execute(obs_plan, [username, orgname, geomvals, params, merge])
+        empty_results = len(geomvals) - len(result)
+        if empty_results > 0:
+          quota_service.increment_empty_service_use(empty_results)
+        if result:
+          quota_service.increment_success_service_use(len(result))
+          return result
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use(len(geomvals))
+        logger.error('Error trying to OBS_GetData', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetData')
+    finally:
+        quota_service.increment_total_service_use(len(geomvals))
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomrefs TEXT[],
+  params JSON)
+RETURNS TABLE (
+  id TEXT,
+  data JSON
+) AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT * FROM cdb_observatory.OBS_GetData(geomrefs, params);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetData(
+  username TEXT,
+  orgname TEXT,
+  geomrefs TEXT[],
+  params JSON)
+RETURNS TABLE (
+  id TEXT,
+  data JSON
+) AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    raise Exception('You have reached the limit of your quota')
+
+  with metrics('obs_getdata', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetData($1, $2, $3, $4);", ["text", "text", "text[]", "json"])
+        result = plpy.execute(obs_plan, [username, orgname, geomrefs, params])
+        empty_results = len(geomrefs) - len(result)
+        if empty_results > 0:
+          quota_service.increment_empty_service_use(empty_results)
+        if result:
+          quota_service.increment_success_service_use(len(result))
+          return result
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        quota_service.increment_failed_service_use(len(geomrefs))
+        exc_info = sys.exc_info()
+        logger.error('%s, %s, %s' % (exc_info[0], exc_info[1], exc_info[2]))
+        logger.error('Error trying to OBS_GetData', exc_info, data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetData')
+    finally:
+        quota_service.increment_total_service_use(len(geomrefs))
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeta(
+  username TEXT,
+  orgname TEXT,
+  geom Geometry(Geometry, 4326),
+  params JSON,
+  max_timespan_rank INTEGER DEFAULT NULL,
+  max_score_rank INTEGER DEFAULT NULL,
+  target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetMeta(geom, params, max_timespan_rank, max_score_rank, target_geoms);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeta(
+  username TEXT,
+  orgname TEXT,
+  geom Geometry(Geometry, 4326),
+  params JSON,
+  max_timespan_rank INTEGER DEFAULT NULL,
+  max_score_rank INTEGER DEFAULT NULL,
+  target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  with metrics('obs_getmeta', user_obs_config, logger):
+    try:
+        obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeta($1, $2, $3, $4, $5, $6, $7) as meta;", ["text", "text", "Geometry (Geometry, 4326)", "json", "integer", "integer", "integer"])
+        result = plpy.execute(obs_plan, [username, orgname, geom, params, max_timespan_rank, max_score_rank, target_geoms])
+        if result:
+          return result[0]['meta']
+        else:
+          return None
+    except BaseException as e:
+        import sys
+        logger.error('Error trying to OBS_GetMeta', sys.exc_info(), data={"username": username, "orgname": orgname})
+        raise Exception('Error trying to OBS_GetMeta')
+$$ LANGUAGE plpythonu;

server/extension/test/expected/100_data_observatory_test.out

@@ -141,6 +141,39 @@ SELECT exists(SELECT *
  t
 (1 row)
 
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getmeta'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, json, integer, integer, integer');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getdata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geomval[], json, boolean');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getdata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text[], json');
+ exists 
+--------
+ t
+(1 row)
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/100_data_observatory_test.sql

@@ -89,6 +89,27 @@ SELECT exists(SELECT *
               AND proname = 'obs_getboundarybyid'
               AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text');
 
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getmeta'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, json, integer, integer, integer');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getdata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, geomval[], json, boolean');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getdata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text[], json');
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/lib/python/cartodb_services/README.md

@@ -46,5 +46,7 @@ cd $(git rev-parse --show-toplevel)/test
 python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY
 ```
 
-## TODO
-- Move dependencies expressed in `requirements.txt` to `setup.py`
+## Versioning
+Once you're satisfied with your changes, it is time to bump the version number in the `setup.py`. A couple of rules:
+- **Backwards compatibility**: in general all changes shall be backwards compatible. Do not remove any code used from the server public `pl/python` functions or you'll run into problems when deploying.
+- **Semantic versioning**: we try to stick to [Semantic Versioning 2.0.0](http://semver.org/spec/v2.0.0.html)

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -4,6 +4,7 @@
 import json
 import requests
 
+from requests.adapters import HTTPAdapter
 from exceptions import *
 from cartodb_services.metrics import Traceable
 
@@ -17,6 +18,7 @@ class HereMapsGeocoder(Traceable):
     DEFAULT_GEN = 9
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ADDRESS_PARAMS = [
         'city',
@@ -88,7 +90,10 @@ class HereMapsGeocoder(Traceable):
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self.host, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 
 from exceptions import WrongParams
+from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 
 
@@ -13,6 +14,7 @@ class HereMapsRoutingIsoline(Traceable):
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -53,6 +55,9 @@ class HereMapsRoutingIsoline(Traceable):
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
         response = requests.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenGeocoder(Traceable):
     BASE_URL = 'https://search.mapzen.com/v1/search'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -27,7 +29,10 @@ class MapzenGeocoder(Traceable):
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
             self.add_response_data(response, self._logger)
             if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 
@@ -12,6 +13,7 @@ class MapzenIsochrones:
     BASE_URL = 'https://matrix.mapzen.com/isochrone'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -23,12 +25,15 @@ class MapzenIsochrones:
         self._url = base_url
         self._logger = logger
 
-    @qps_retry
+    @qps_retry(qps=7)
     def isochrone(self, locations, costing, ranges):
         request_params = self._parse_request_params(locations, costing,
                                                     ranges)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT,
                                             self.READ_TIMEOUT))
 

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -55,7 +55,7 @@ class QPSService:
 
     def retry(self, first_request_time, retry_count):
         elapsed = datetime.now() - first_request_time
-        if elapsed.microseconds  > (self._retry_timeout * 1000.0):
+        if elapsed.total_seconds()  > self._retry_timeout:
             raise TimeoutException()
 
         # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenRouting(Traceable):
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -46,7 +48,10 @@ class MapzenRouting(Traceable):
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -43,12 +43,20 @@ class ServiceConfig(object):
     def metrics_log_path(self):
         return self._metrics_log_path
 
+    def _get_effective_monthly_quota(self, quota_key, default=0):
+        quota_from_redis = self._redis_config.get(quota_key, None)
+        if quota_from_redis and quota_from_redis <> '':
+            return int(quota_from_redis)
+        else:
+            return default
+
     def __get_metrics_log_path(self):
         if self.METRICS_LOG_KEY:
             return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None)
         else:
             return None
 
+
 class DataObservatoryConfig(ServiceConfig):
 
     METRICS_LOG_KEY = 'do_log_path'
@@ -92,9 +100,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig):
             self._soft_limit = True
         else:
             self._soft_limit = False
-        self._monthly_quota = 0
-        if self.QUOTA_KEY in self._redis_config:
-            self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
+        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._connection_str = self._db_config.data_observatory_connection_str
 
     @property
@@ -116,9 +122,7 @@ class ObservatoryConfig(DataObservatoryConfig):
             self._soft_limit = True
         else:
             self._soft_limit = False
-        self._monthly_quota = 0
-        if self.QUOTA_KEY in self._redis_config:
-            self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
+        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._connection_str = self._db_config.data_observatory_connection_str
 
     @property
@@ -173,14 +177,7 @@ class RoutingConfig(ServiceConfig):
         return self._soft_limit
 
     def _set_monthly_quota(self):
-        self._monthly_quota = self._get_effective_monthly_quota()
-
-    def _get_effective_monthly_quota(self):
-        quota_from_redis = self._redis_config.get(self.QUOTA_KEY)
-        if quota_from_redis and quota_from_redis <> '':
-            return int(quota_from_redis)
-        else:
-            return self._db_config.mapzen_routing_monthly_quota
+        self._monthly_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
 
     def _set_soft_limit(self):
         if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
@@ -202,7 +199,7 @@ class IsolinesRoutingConfig(ServiceConfig):
     GEOCODER_PROVIDER_KEY = 'geocoder_provider'
     MAPZEN_PROVIDER = 'mapzen'
     HEREMAPS_PROVIDER = 'heremaps'
-    DEFAULT_PROVIDER = 'heremaps'
+    DEFAULT_PROVIDER = 'mapzen'
     METRICS_LOG_KEY = 'isolines_log_path'
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -217,18 +214,16 @@ class IsolinesRoutingConfig(ServiceConfig):
             self._isolines_provider = self.DEFAULT_PROVIDER
         self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
+        self._isolines_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
+        if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
+            self._soft_isolines_limit = True
+        else:
+            self._soft_isolines_limit = False
         if self._isolines_provider == self.HEREMAPS_PROVIDER:
-            self._isolines_quota = int(filtered_config[self.QUOTA_KEY])
             self._heremaps_app_id = db_config.heremaps_isolines_app_id
             self._heremaps_app_code = db_config.heremaps_isolines_app_code
-            if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
-                self._soft_isolines_limit = True
-            else:
-                self._soft_isolines_limit = False
         elif self._isolines_provider == self.MAPZEN_PROVIDER:
             self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
-            self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
-            self._soft_isolines_limit = False
 
     @property
     def service_type(self):
@@ -361,8 +356,10 @@ class GeocoderConfig(ServiceConfig):
             self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
         else:
             self._geocoder_provider = self.DEFAULT_PROVIDER
-        self._geocoding_quota = int(filtered_config[self.QUOTA_KEY])
+
+        self._geocoding_quota = self._get_effective_monthly_quota(self.QUOTA_KEY)
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
+
         if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_geocoding_limit = True
         else:
@@ -377,7 +374,6 @@ class GeocoderConfig(ServiceConfig):
             self._cost_per_hit = 0
         elif self._geocoder_provider == self.MAPZEN_GEOCODER:
             self._mapzen_api_key = db_config.mapzen_geocoder_api_key
-            self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota
             self._cost_per_hit = 0
 
     @property

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -1,5 +1,17 @@
 from datetime import date, timedelta
 from dateutil.relativedelta import relativedelta
+from calendar import monthrange
+
+
+def last_day_of_month(year, month):
+    """last valid day of a month"""
+    return monthrange(year, month)[1]
+
+
+def latest_valid_date(year, month, day):
+    """latest date not later than the day specified"""
+    valid_day = min(day, last_day_of_month(year, month))
+    return date(year, month, valid_day)
 
 
 class UserMetricsService:
@@ -8,6 +20,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
     SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
     SERVICE_OBSERVATORY = 'obs_general'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
@@ -21,6 +34,8 @@ class UserMetricsService:
     def used_quota(self, service_type, date):
         if service_type == self.SERVICE_HERE_ISOLINES:
             return self.__used_isolines_quota(service_type, date)
+        if service_type == self.SERVICE_MAPZEN_ISOLINES:
+            return self.__used_isolines_quota(service_type, date)
         elif service_type == self.SERVICE_MAPZEN_ROUTING:
             return self.__used_routing_quota(service_type, date)
         elif service_type == self.SERVICE_OBSERVATORY:
@@ -143,9 +158,9 @@ class UserMetricsService:
         today = date.today()
         if end_period_day > today.day:
             temp_date = today + relativedelta(months=-1)
-            date_from = date(temp_date.year, temp_date.month, end_period_day)
+            date_from = latest_valid_date(temp_date.year, temp_date.month, end_period_day)
         else:
-            date_from = date(today.year, today.month, end_period_day)
+            date_from = latest_valid_date(today.year, today.month, end_period_day)
 
         return date_from, today
 

server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py

@@ -67,6 +67,7 @@ class MapzenGeocoderConfig(object):
     @property
     def username(self):
         return self._username
+
     @property
     def organization(self):
         return self._organization
@@ -86,16 +87,13 @@ class MapzenGeocoderConfigBuilder(object):
         self._username = username
         self._orgname = orgname
 
-
     def get(self):
         mapzen_server_conf = self._server_conf.get('mapzen_conf')
-        geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota']
         mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
 
+        geocoding_quota = self._get_quota(mapzen_server_conf)
         soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
-
-        cost_per_hit=0
-
+        cost_per_hit = 0
         period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
         period_end_date = date_parse(period_end_date_str)
 
@@ -110,3 +108,10 @@ class MapzenGeocoderConfigBuilder(object):
                                     mapzen_api_key,
                                     self._username,
                                     self._orgname)
+
+    def _get_quota(self, mapzen_server_conf):
+        geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
+        if geocoding_quota is '':
+            return 0
+
+        return int(geocoding_quota)

server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py

@@ -41,7 +41,7 @@ class RedisDBConfig:
             key)
         conf = self._db_conn.execute(conf_query)[0]['conf']
         if conf is None:
-            raise "There is no redis configuration defined"
+            raise Exception("There is no redis configuration defined")
         else:
             params = json.loads(conf)
             self._host = params['redis_host']

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.12.0',
+    version='0.13.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -1,7 +1,282 @@
 from unittest import TestCase
 from mockredis import MockRedis
+from datetime import datetime, timedelta
 from ..test_helper import *
-from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
+from cartodb_services.metrics.config import *
+
+
+class TestGeocoderUserConfig(TestCase):
+
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_geocoder_config_for_user(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    provider=geocoder_provider, quota=100)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            if geocoder_provider == 'heremaps':
+                assert geocoder_config.heremaps_geocoder is True
+                assert geocoder_config.geocoding_quota == 100
+            elif geocoder_provider == 'mapzen':
+                assert geocoder_config.mapzen_geocoder is True
+                assert geocoder_config.geocoding_quota == 100
+            elif geocoder_provider == 'google':
+                assert geocoder_config.google_geocoder is True
+                assert geocoder_config.geocoding_quota is None
+            assert geocoder_config.soft_geocoding_limit is False
+
+    def test_should_return_quota_0_when_is_0_in_redis(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota=0, provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            if geocoder_provider is not 'google':
+                assert geocoder_config.geocoding_quota == 0
+
+    def test_should_return_quota_0_if_quota_is_empty(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota='', provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            if geocoder_provider is not 'google':
+                assert geocoder_config.geocoding_quota == 0
+
+    def test_should_return_quota_None_when_is_provider_is_google(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota=0, provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            if geocoder_provider is 'google':
+                assert geocoder_config.geocoding_quota == None
+
+    def test_should_return_true_if_soft_limit_is_true(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota=0, soft_limit=True,
+                                    provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            assert geocoder_config.soft_geocoding_limit == True
+
+    def test_should_return_false_if_soft_limit_is_empty_string(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota=0, soft_limit='',
+                                    provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', None)
+            assert geocoder_config.soft_geocoding_limit == False
+
+class TestGeocoderOrgConfig(TestCase):
+
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'google']
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_org_config(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            yesterday = datetime.today() - timedelta(days=1)
+
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    provider=geocoder_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
+                                   quota=200, end_date=yesterday,
+                                   provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', 'test_org')
+            if geocoder_provider == 'heremaps':
+                assert geocoder_config.heremaps_geocoder is True
+                assert geocoder_config.geocoding_quota == 200
+            elif geocoder_provider == 'mapzen':
+                assert geocoder_config.mapzen_geocoder is True
+                assert geocoder_config.geocoding_quota == 200
+            elif geocoder_provider == 'google':
+                assert geocoder_config.google_geocoder is True
+                assert geocoder_config.geocoding_quota is None
+            assert geocoder_config.soft_geocoding_limit is False
+            assert geocoder_config.period_end_date.date() == yesterday.date()
+
+    def test_should_return_0_quota_if_has_0_in_redis_config(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            yesterday = datetime.today() - timedelta(days=1)
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    provider=geocoder_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
+                                   quota=0, end_date=yesterday,
+                                   provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                            'test_user', 'test_org')
+            if geocoder_provider is not 'google':
+                assert geocoder_config.geocoding_quota == 0
+
+    def test_should_return_0_if_quota_is_empty_for_org_in_redis(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            yesterday = datetime.today() - timedelta(days=1)
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    provider=geocoder_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
+                                   quota='', end_date=yesterday,
+                                   provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                         'test_user', 'test_org')
+            if geocoder_provider is not 'google':
+                assert geocoder_config.geocoding_quota == 0
+
+    def test_should_return_None_if_provider_is_google(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            yesterday = datetime.today() - timedelta(days=1)
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    provider=geocoder_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
+                                   quota='', end_date=yesterday,
+                                   provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                         'test_user', 'test_org')
+            if geocoder_provider is 'google':
+                assert geocoder_config.geocoding_quota == None
+
+    def test_should_return_user_quota_if_is_not_defined_for_org(self):
+        for geocoder_provider in self.GEOCODER_PROVIDERS:
+            yesterday = datetime.today() - timedelta(days=1)
+            build_redis_user_config(self.redis_conn, 'test_user', 'geocoding',
+                                    quota=100, provider=geocoder_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'geocoding',
+                                   quota=None, end_date=yesterday,
+                                   provider=geocoder_provider)
+            geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
+                                         'test_user', 'test_org')
+            if geocoder_provider is not 'google':
+                assert geocoder_config.geocoding_quota == 100
+
+
+class TestIsolinesUserConfig(TestCase):
+
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_user_config_for_isolines(self):
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    quota=100, provider=isolines_provider)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user')
+            if isolines_provider is 'mapzen':
+                assert isolines_config.service_type is 'mapzen_isolines'
+            else:
+                assert isolines_config.service_type is 'here_isolines'
+            assert isolines_config.isolines_quota == 100
+            assert isolines_config.soft_isolines_limit is False
+
+    def test_should_return_0_quota_for_0_value(self):
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider, quota=0,
+                                    soft_limit=True)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user')
+            assert isolines_config.isolines_quota == 0
+
+    def test_should_return_0_quota_for_empty_quota_value(self):
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider, quota='')
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user')
+            assert isolines_config.isolines_quota == 0
+
+    def test_should_return_true_soft_limit(self):
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider, quota=0,
+                                    soft_limit=True)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user')
+            assert isolines_config.soft_isolines_limit is True
+
+    def test_should_return_false_soft_limit_with_empty_string(self):
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider, quota=0,
+                                    soft_limit='')
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user')
+            assert isolines_config.soft_isolines_limit is False
+
+class TestIsolinesOrgConfig(TestCase):
+
+    ISOLINES_PROVIDERS = ['heremaps', 'mapzen']
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_org_config_for_isolines(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
+                                   quota=200, end_date=yesterday,
+                                   provider=isolines_provider)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user', 'test_org')
+            assert isolines_config.isolines_quota == 200
+            assert isolines_config.soft_isolines_limit is False
+            assert isolines_config.period_end_date.date() == yesterday.date()
+
+    def test_should_return_quota_0_for_0_redis_quota(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider,
+                                    soft_limit=True)
+            build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
+                                   quota=0, end_date=yesterday,
+                                   provider=isolines_provider)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user', 'test_org')
+            assert isolines_config.isolines_quota == 0
+
+    def test_should_return_quota_0_for_empty_string_quota_in_org_config(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider)
+            build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
+                                   quota='', end_date=yesterday,
+                                   provider=isolines_provider)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user', 'test_org')
+            assert isolines_config.isolines_quota == 0
+
+    def test_should_return_user_quota_for_non_existent_org_quota(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        for isolines_provider in self.ISOLINES_PROVIDERS:
+
+            build_redis_user_config(self.redis_conn, 'test_user', 'isolines',
+                                    provider=isolines_provider, quota=100)
+            build_redis_org_config(self.redis_conn, 'test_org', 'isolines',
+                                   quota=None, end_date=yesterday,
+                                   provider=isolines_provider)
+            isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
+                                                    'test_user', 'test_org')
+            assert isolines_config.isolines_quota == 100
 
 
 class TestRoutingConfig(TestCase):
@@ -13,11 +288,6 @@ class TestRoutingConfig(TestCase):
         self._user_key = "rails:users:{0}".format(self._username)
         self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')
 
-    def test_should_pick_quota_from_server_by_default(self):
-        orgname = None
-        config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
-        assert config.monthly_quota == 1500000
-
     def test_should_pick_quota_from_redis_if_present(self):
         self._redis_conn.hset(self._user_key, 'mapzen_routing_quota', 1000)
         orgname = None
@@ -38,7 +308,6 @@ class TestRoutingConfig(TestCase):
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
         assert config.monthly_quota == 5000
 
-
     def test_should_have_soft_limit_false_by_default(self):
         orgname = None
         config = RoutingConfig(self._redis_conn, self._db_conn, self._username, orgname)
@@ -51,6 +320,136 @@ class TestRoutingConfig(TestCase):
         assert config.soft_limit == True
 
 
+class TestDataObservatoryUserConfig(TestCase):
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_config_for_obs_snapshot(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100, end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
+                                          'test_user')
+        assert do_config.monthly_quota == 100
+        assert do_config.soft_limit is False
+        assert do_config.period_end_date.date() == yesterday.date()
+
+    def test_should_return_true_if_soft_limit_is_true_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=0, soft_limit=True, end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
+                                          'test_user')
+        assert do_config.soft_limit is True
+
+    def test_should_return_0_if_quota_is_0_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=0, end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
+                                              'test_user')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_0_if_quota_is_empty_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota='', end_date=yesterday)
+        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
+                                              'test_user')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_config_for_obs_snapshot(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                          'test_user')
+        assert do_config.monthly_quota == 100
+        assert do_config.soft_limit is False
+        assert do_config.period_end_date.date() == yesterday.date()
+
+    def test_should_return_0_if_quota_is_0_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=0, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                              'test_user')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_0_if_quota_is_empty_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota='', end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                              'test_user')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_true_if_soft_limit_is_true_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=0, soft_limit=True, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                          'test_user')
+        assert do_config.soft_limit is True
+
+    def test_should_return_true_if_soft_limit_is_empty_string_in_redis(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=0, soft_limit='', end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                          'test_user')
+        assert do_config.soft_limit is False
+
+class TestDataObservatoryOrgConfig(TestCase):
+
+    def setUp(self):
+        self.redis_conn = MockRedis()
+        plpy_mock_config()
+
+    def test_should_return_organization_config(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100, end_date=yesterday)
+        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
+                                quota=200, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                      'test_user', 'test_org')
+        assert do_config.monthly_quota == 200
+        assert do_config.period_end_date.date() == yesterday.date()
+
+    def test_should_return_quota_0_for_0_in_org_quota_config(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100)
+        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
+                                quota=0, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                      'test_user', 'test_org')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_quota_0_for_empty_in_org_quota_config(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100)
+        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
+                                quota='', end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                      'test_user', 'test_org')
+        assert do_config.monthly_quota == 0
+
+    def test_should_return_user_config_when_org_quota_is_not_defined(self):
+        yesterday = datetime.today() - timedelta(days=1)
+        build_redis_user_config(self.redis_conn, 'test_user', 'data_observatory',
+                                quota=100)
+        build_redis_org_config(self.redis_conn, 'test_org', 'data_observatory',
+                                quota=None, end_date=yesterday)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                      'test_user', 'test_org')
+        assert do_config.monthly_quota == 100
+
+
 class TestServicesRedisConfig(TestCase):
     def test_it_picks_mapzen_routing_quota_from_redis(self):
         redis_conn = MockRedis()

server/lib/python/cartodb_services/test/metrics/test_quota.py

@@ -22,7 +22,7 @@ class TestQuotaChecker(TestCase):
             self.username,
             self.service_type,
             self.period_end_date.year,
-            self.period_end_date.month
+            self.period_end_date.strftime('%m')
         )
 
     def test_routing_quota_check_passes_when_enough_quota(self):

server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py

@@ -0,0 +1,154 @@
+from unittest import TestCase
+from mockredis import MockRedis
+from datetime import datetime
+from cartodb_services.refactor.service.mapzen_geocoder_config import *
+from cartodb_services.refactor.storage.redis_config import *
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+
+
+class TestMapzenGeocoderUserConfig(TestCase):
+
+    def setUp(self):
+        self._redis_connection = MockRedis()
+        self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
+                                                     "mapzen_conf":
+                                                     {"geocoder":
+                                                      {"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
+                                                     }, "logger_conf": {}})
+        self._username = 'test_user'
+        self._user_key = "rails:users:{0}".format(self._username)
+        self._user_config = RedisUserConfigStorageBuilder(self._redis_connection,
+                                                    self._username).get()
+        self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection,
+                                                  None).get()
+        self._set_default_config_values()
+
+    def test_config_values_are_ok(self):
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             None).get()
+        assert config.geocoding_quota == 100
+        assert config.soft_geocoding_limit == False
+        assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S")
+        assert config.service_type == 'geocoder_mapzen'
+        assert config.provider == 'mapzen'
+        assert config.is_high_resolution == True
+        assert config.cost_per_hit == 0
+        assert config.mapzen_api_key == 'search-xxxxxxx'
+        assert config.username == 'test_user'
+        assert config.organization is None
+
+    def test_quota_should_be_0_if_redis_value_is_0(self):
+        self._redis_connection.hset(self._user_key, 'geocoding_quota', '0')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             None).get()
+        assert config.geocoding_quota == 0
+
+    def test_quota_should_be_0_if_redis_value_is_empty_string(self):
+        self._redis_connection.hset(self._user_key, 'geocoding_quota', '')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             None).get()
+        assert config.geocoding_quota == 0
+
+    def test_soft_limit_should_be_true(self):
+        self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'true')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             None).get()
+        assert config.soft_geocoding_limit == True
+
+    def test_soft_limit_should_be_false_if_is_empty_string(self):
+        self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', '')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             None).get()
+        assert config.soft_geocoding_limit == False
+
+    def _set_default_config_values(self):
+        self._redis_connection.hset(self._user_key, 'geocoding_quota', '100')
+        self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
+        self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00')
+
+class TestMapzenGeocoderOrgConfig(TestCase):
+
+    def setUp(self):
+        self._redis_connection = MockRedis()
+        self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
+                                                     "mapzen_conf":
+                                                     {"geocoder":
+                                                      {"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
+                                                     }, "logger_conf": {}})
+        self._username = 'test_user'
+        self._organization = 'test_org'
+        self._user_key = "rails:users:{0}".format(self._username)
+        self._org_key = "rails:orgs:{0}".format(self._organization)
+        self._user_config = RedisUserConfigStorageBuilder(self._redis_connection,
+                                                    self._username).get()
+        self._org_config = RedisOrgConfigStorageBuilder(self._redis_connection,
+                                                  self._organization).get()
+        self._set_default_config_values()
+
+
+    def test_config_org_values_are_ok(self):
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             self._organization).get()
+        assert config.geocoding_quota == 200
+        assert config.soft_geocoding_limit == False
+        assert config.period_end_date == datetime.strptime('2016-12-31 00:00:00', "%Y-%m-%d %H:%M:%S")
+        assert config.service_type == 'geocoder_mapzen'
+        assert config.provider == 'mapzen'
+        assert config.is_high_resolution == True
+        assert config.cost_per_hit == 0
+        assert config.mapzen_api_key == 'search-xxxxxxx'
+        assert config.username == 'test_user'
+        assert config.organization is 'test_org'
+
+    def test_quota_should_be_0_if_redis_value_is_0(self):
+        self._redis_connection.hset(self._org_key, 'geocoding_quota', '0')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             self._organization).get()
+        assert config.geocoding_quota == 0
+
+    def test_quota_should_use_user_quota_value_if_redis_value_is_empty_string(self):
+        self._redis_connection.hset(self._org_key, 'geocoding_quota', '')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             self._organization).get()
+        assert config.geocoding_quota == 100
+
+    def test_quota_should_be_0_if_both_user_and_org_have_empty_string(self):
+        self._redis_connection.hset(self._user_key, 'geocoding_quota', '')
+        self._redis_connection.hset(self._org_key, 'geocoding_quota', '')
+        config = MapzenGeocoderConfigBuilder(self._server_config,
+                                             self._user_config,
+                                             self._org_config,
+                                             self._username,
+                                             self._organization).get()
+        assert config.geocoding_quota == 0
+
+    def _set_default_config_values(self):
+        self._redis_connection.hset(self._user_key, 'geocoding_quota', '100')
+        self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
+        self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00')
+        self._redis_connection.hset(self._org_key, 'geocoding_quota', '200')
+        self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00')

server/lib/python/cartodb_services/test/test_config.py

@@ -1,60 +0,0 @@
-from test_helper import *
-from unittest import TestCase
-from nose.tools import assert_raises
-from mockredis import MockRedis
-from datetime import datetime, timedelta
-from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
-
-
-class TestConfig(TestCase):
-
-    def setUp(self):
-        self.redis_conn = MockRedis()
-        plpy_mock_config()
-
-    def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
-        build_redis_user_config(self.redis_conn, 'test_user')
-        geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
-                                         'test_user', None)
-        assert geocoder_config.heremaps_geocoder is True
-        assert geocoder_config.geocoding_quota == 100
-        assert geocoder_config.soft_geocoding_limit is False
-
-    def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user')
-        build_redis_org_config(self.redis_conn, 'test_org',
-                               quota=200, end_date=yesterday)
-        geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
-                                         'test_user', 'test_org')
-        assert geocoder_config.heremaps_geocoder is True
-        assert geocoder_config.geocoding_quota == 200
-        assert geocoder_config.soft_geocoding_limit is False
-        assert geocoder_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_config_for_obs_snapshot(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user',
-                                do_quota=100, soft_do_limit=True,
-                                end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.monthly_quota == 100
-        assert do_config.soft_limit is True
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
-        yesterday = datetime.today() - timedelta(days=1)
-        build_redis_user_config(self.redis_conn, 'test_user',
-                                end_date=yesterday)
-        do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
-                                          'test_user')
-        assert do_config.monthly_quota == 0
-        assert do_config.soft_limit is False
-        assert do_config.period_end_date.date() == yesterday.date()
-
-    def test_should_raise_exception_when_missing_parameters(self):
-        plpy_mock._reset()
-        build_redis_user_config(self.redis_conn, 'test_user')
-        assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
-                      plpy_mock, 'test_user', None)

server/lib/python/cartodb_services/test/test_helper.py

@@ -1,4 +1,5 @@
 from datetime import datetime, date
+from dateutil.tz import tzlocal
 from mock import Mock, MagicMock
 import random
 import sys
@@ -8,46 +9,60 @@ plpy_mock = MockPlPy()
 sys.modules['plpy'] = plpy_mock
 
 
-def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
-                            service="heremaps", isolines_quota=0,
-                            do_quota=None, soft_do_limit=None,
-                            do_general_quota=None, soft_do_general_limit=None,
+def build_redis_user_config(redis_conn, username, service, quota=100,
+                            soft_limit=False, provider="heremaps",
                             end_date=datetime.today()):
+    end_date_tz = end_date.replace(tzinfo=tzlocal())
     user_redis_name = "rails:users:{0}".format(username)
-    redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
-    redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
-    redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota)
-    redis_conn.hset(user_redis_name, 'geocoder_provider', service)
-    redis_conn.hset(user_redis_name, 'isolines_provider', service)
-    redis_conn.hset(user_redis_name, 'routing_provider', service)
-    redis_conn.hset(user_redis_name, 'period_end_date', end_date)
-    if do_quota:
-        redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota)
-    if soft_do_limit:
-        redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit',
-                        soft_do_limit)
-    if do_general_quota:
-        redis_conn.hset(user_redis_name, 'obs_general_quota', do_general_quota)
-    if soft_do_general_limit:
-        redis_conn.hset(user_redis_name, 'soft_obs_general_limit',
-                        soft_do_general_limit)
+
+    if service is 'geocoding':
+        redis_conn.hset(user_redis_name, 'geocoder_provider', provider)
+        redis_conn.hset(user_redis_name, 'geocoding_quota', str(quota))
+        redis_conn.hset(user_redis_name, 'soft_geocoding_limit', str(soft_limit).lower())
+    elif service is 'isolines':
+        redis_conn.hset(user_redis_name, 'isolines_provider', provider)
+        redis_conn.hset(user_redis_name, 'here_isolines_quota', str(quota))
+        redis_conn.hset(user_redis_name, 'soft_here_isolines_limit', str(soft_limit).lower())
+    elif service is 'routing':
+        redis_conn.hset(user_redis_name, 'routing_provider', provider)
+        redis_conn.hset(user_redis_name, 'mapzen_routing_quota', str(quota))
+        redis_conn.hset(user_redis_name, 'soft_mapzen_routing_limit', str(soft_limit).lower())
+    elif service is 'data_observatory':
+        redis_conn.hset(user_redis_name, 'obs_snapshot_quota', str(quota))
+        redis_conn.hset(user_redis_name, 'obs_general_quota', str(quota))
+        redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', str(soft_limit).lower())
+        redis_conn.hset(user_redis_name, 'soft_obs_general_limit', str(soft_limit).lower())
+
     redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
     redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
+    redis_conn.hset(user_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z"))
 
 
-def build_redis_org_config(redis_conn, orgname, quota=100, service="heremaps",
-                           isolines_quota=0, do_quota=None,
-                           do_general_quota=None, end_date=datetime.today()):
+def build_redis_org_config(redis_conn, orgname, service, quota=100,
+                           provider="heremaps", end_date=datetime.now(tzlocal())):
     org_redis_name = "rails:orgs:{0}".format(orgname)
-    redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
-    redis_conn.hset(org_redis_name, 'here_isolines_quota', isolines_quota)
-    if do_quota:
-        redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota)
-    if do_general_quota:
-        redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota)
-    redis_conn.hset(org_redis_name, 'period_end_date', end_date)
+    end_date_tz = end_date.replace(tzinfo=tzlocal())
+
+    if service is 'geocoding':
+        redis_conn.hset(org_redis_name, 'geocoder_provider', provider)
+        if quota is not None:
+            redis_conn.hset(org_redis_name, 'geocoding_quota', str(quota))
+    elif service is 'isolines':
+        redis_conn.hset(org_redis_name, 'isolines_provider', provider)
+        if quota is not None:
+            redis_conn.hset(org_redis_name, 'here_isolines_quota', str(quota))
+    elif service is 'routing':
+        redis_conn.hset(org_redis_name, 'routing_provider', provider)
+        if quota is not None:
+            redis_conn.hset(org_redis_name, 'mapzen_routing_quota', str(quota))
+    elif service is 'data_observatory':
+        if quota is not None:
+            redis_conn.hset(org_redis_name, 'obs_snapshot_quota', str(quota))
+            redis_conn.hset(org_redis_name, 'obs_general_quota', str(quota))
+
     redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
     redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
+    redis_conn.hset(org_redis_name, 'period_end_date', end_date_tz.strftime("%Y-%m-%d %H:%M:%S %z"))
 
 
 def increment_service_uses(redis_conn, username, orgname=None,

server/lib/python/cartodb_services/test/test_quota_service.py

@@ -1,7 +1,7 @@
 from test_helper import *
 from mockredis import MockRedis
 from cartodb_services.metrics import QuotaService
-from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
+from cartodb_services.metrics import *
 from unittest import TestCase
 from nose.tools import assert_raises
 from datetime import datetime, date
@@ -35,26 +35,26 @@ class TestQuotaService(TestCase):
         qs = self.__build_geocoder_quota_service('test_user',
                                                  orgname='test_org')
         increment_service_uses(self.redis_conn, 'test_user',
-                                           orgname='test_org')
+                               orgname='test_org')
         assert qs.check_user_quota() is True
 
     def test_should_return_false_if_user_quota_is_surpassed(self):
         qs = self.__build_geocoder_quota_service('test_user')
         increment_service_uses(self.redis_conn, 'test_user',
-                                           amount=300)
+                               amount=300)
         assert qs.check_user_quota() is False
 
     def test_should_return_false_if_org_quota_is_surpassed(self):
         qs = self.__build_geocoder_quota_service('test_user',
                                                  orgname='test_org')
         increment_service_uses(self.redis_conn, 'test_user',
-                                           orgname='test_org', amount=400)
+                               orgname='test_org', amount=400)
         assert qs.check_user_quota() is False
 
     def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
         qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
         increment_service_uses(self.redis_conn, 'test_user',
-                                           amount=300)
+                               amount=300)
         assert qs.check_user_quota() is True
 
     def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
@@ -62,7 +62,7 @@ class TestQuotaService(TestCase):
                                                  orgname='test_org',
                                                  soft_limit=True)
         increment_service_uses(self.redis_conn, 'test_user',
-                                            orgname='test_org', amount=400)
+                               orgname='test_org', amount=400)
         assert qs.check_user_quota() is True
 
     def test_should_check_user_increment_and_quota_check_correctly(self):
@@ -81,15 +81,17 @@ class TestQuotaService(TestCase):
         assert qs.check_user_quota() is False
 
     def test_should_check_user_mapzen_geocoder_quota_correctly(self):
-        qs = self.__build_geocoder_quota_service('test_user', service='mapzen')
+        qs = self.__build_geocoder_quota_service('test_user',
+                                                 provider='mapzen')
         qs.increment_success_service_use()
         assert qs.check_user_quota() is True
         qs.increment_success_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
     def test_should_check_org_mapzen_geocoder_quota_correctly(self):
-        qs = self.__build_geocoder_quota_service('test_user', orgname='testorg',
-                                                service='mapzen')
+        qs = self.__build_geocoder_quota_service('test_user',
+                                                 orgname='testorg',
+                                                 provider='mapzen')
         qs.increment_success_service_use()
         assert qs.check_user_quota() is True
         qs.increment_success_service_use(amount=1500000)
@@ -111,16 +113,17 @@ class TestQuotaService(TestCase):
 
     def test_should_check_user_isolines_quota_correctly(self):
         qs = self.__build_isolines_quota_service('test_user')
-        qs.increment_success_service_use()
+        qs.increment_isolines_service_use()
         assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=1500000)
+        qs.increment_isolines_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
     def test_should_check_org_isolines_quota_correctly(self):
-        qs = self.__build_isolines_quota_service('test_user', orgname='testorg')
-        qs.increment_success_service_use()
+        qs = self.__build_isolines_quota_service('test_user',
+                                                 orgname='testorg')
+        qs.increment_isolines_service_use()
         assert qs.check_user_quota() is True
-        qs.increment_success_service_use(amount=1500000)
+        qs.increment_isolines_service_use(amount=1500000)
         assert qs.check_user_quota() is False
 
     def test_should_check_user_obs_snapshot_quota_correctly(self):
@@ -138,55 +141,77 @@ class TestQuotaService(TestCase):
         qs.increment_success_service_use(amount=100000)
         assert qs.check_user_quota() is False
 
-    def __prepare_quota_service(self, username, quota, service, orgname,
-                                soft_limit, do_quota, soft_do_limit, end_date):
-        build_redis_user_config(self.redis_conn, username,
-                                            quota=quota, service=service,
-                                            soft_limit=soft_limit,
-                                            soft_do_limit=soft_do_limit,
-                                            do_quota=do_quota,
-                                            end_date=end_date)
+    def test_should_check_user_obs_quota_correctly(self):
+        qs = self.__build_obs_snapshot_quota_service('test_user')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
+    def test_should_check_org_obs_quota_correctly(self):
+        qs = self.__build_obs_quota_service('test_user',
+                                            orgname='testorg')
+        qs.increment_success_service_use()
+        assert qs.check_user_quota() is True
+        qs.increment_success_service_use(amount=100000)
+        assert qs.check_user_quota() is False
+
+    def __prepare_quota_service(self, username, service, quota, provider,
+                                orgname, soft_limit, end_date):
+        build_redis_user_config(self.redis_conn, username, service,
+                                quota=quota, provider=provider,
+                                soft_limit=soft_limit, end_date=end_date)
         if orgname:
-            build_redis_org_config(self.redis_conn, orgname,
-                                               quota=quota, service=service,
-                                               do_quota=do_quota,
-                                               end_date=end_date)
+            build_redis_org_config(self.redis_conn, orgname, service,
+                                   quota=quota, provider=provider,
+                                   end_date=end_date)
 
     def __build_geocoder_quota_service(self, username, quota=100,
-                                       service='heremaps', orgname=None,
+                                       provider='heremaps', orgname=None,
                                        soft_limit=False,
                                        end_date=datetime.today()):
-        self.__prepare_quota_service(username, quota, service, orgname,
-                                     soft_limit, 0, False, end_date)
+        self.__prepare_quota_service(username, 'geocoding', quota,
+                                     provider, orgname, soft_limit, end_date)
         geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          username, orgname)
         return QuotaService(geocoder_config, redis_connection=self.redis_conn)
 
-    def __build_routing_quota_service(self, username, service='mapzen',
+    def __build_routing_quota_service(self, username, provider='mapzen',
                                       orgname=None, soft_limit=False,
                                       quota=100, end_date=datetime.today()):
-        self.__prepare_quota_service(username, quota, service, orgname,
-                                     soft_limit, 0, False, end_date)
+        self.__prepare_quota_service(username, 'routing', quota, provider,
+                                     orgname, soft_limit, end_date)
         routing_config = RoutingConfig(self.redis_conn, plpy_mock,
                                        username, orgname)
         return QuotaService(routing_config, redis_connection=self.redis_conn)
 
-    def __build_isolines_quota_service(self, username, service='mapzen',
+    def __build_isolines_quota_service(self, username, provider='mapzen',
                                       orgname=None, soft_limit=False,
                                       quota=100, end_date=datetime.today()):
-        self.__prepare_quota_service(username, quota, service, orgname,
-                                     soft_limit, 0, False, end_date)
+        self.__prepare_quota_service(username, 'isolines', quota, provider,
+                                     orgname, soft_limit, end_date)
         isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
                                                username, orgname)
         return QuotaService(isolines_config, redis_connection=self.redis_conn)
 
     def __build_obs_snapshot_quota_service(self, username, quota=100,
-                                               service='obs_snapshot',
-                                               orgname=None,
-                                               soft_limit=False,
-                                               end_date=datetime.today()):
-        self.__prepare_quota_service(username, 0, service, orgname, False,
-                                     quota, soft_limit, end_date)
+                                           provider='obs_snapshot',
+                                           orgname=None,
+                                           soft_limit=False,
+                                           end_date=datetime.today()):
+        self.__prepare_quota_service(username, 'data_observatory', quota,
+                                     None, orgname, soft_limit, end_date)
         do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
                                           username, orgname)
         return QuotaService(do_config, redis_connection=self.redis_conn)
+
+    def __build_obs_quota_service(self, username, quota=100,
+                                  provider='obs_general',
+                                  orgname=None,
+                                  soft_limit=False,
+                                  end_date=datetime.today()):
+        self.__prepare_quota_service(username, 'data_observatory', quota,
+                                     None, orgname, soft_limit, end_date)
+        do_config = ObservatoryConfig(self.redis_conn, plpy_mock,
+                                          username, orgname)
+        return QuotaService(do_config, redis_connection=self.redis_conn)

server/lib/python/cartodb_services/test/test_user_service.py

@@ -23,6 +23,10 @@ class TestUserService(TestCase):
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
 
+    def test_user_quota_for_a_month_shorter_than_end_day(self):
+        us = self.__build_user_service('test_user', end_date=datetime(2016,1,31))
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
+
     def test_org_used_quota_for_a_day(self):
         us = self.__build_user_service('test_user', orgname='test_org')
         increment_service_uses(self.redis_conn, 'test_user',
@@ -30,6 +34,10 @@ class TestUserService(TestCase):
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
 
+    def test_org_quota_quota_for_a_month_shorter_than_end_day(self):
+        us = self.__build_user_service('test_user', orgname='test_org', end_date=datetime(2016,1,31))
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
+
     def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
         us = self.__build_user_service('test_user')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0
@@ -108,7 +116,7 @@ class TestUserService(TestCase):
             def zscore_counter(self):
                 return self._zscore_counter
         self.redis_conn = MockRedisWithCounter()
-        us = self.__build_user_service('test_user', end_date=date.today())
+        us = self.__build_user_service('test_user', end_date=datetime.today())
         us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15))
 
         #('user:test_user:geocoder_here:success_responses:201506', 15)
@@ -132,16 +140,17 @@ class TestUserService(TestCase):
         assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506',  '1') == None
 
 
-    def __build_user_service(self, username, quota=100, service='heremaps',
-                             orgname=None, soft_limit=False,
-                             end_date=date.today()):
-        build_redis_user_config(self.redis_conn, username,
-                                            quota=quota, service=service,
-                                            soft_limit=soft_limit,
-                                            end_date=end_date)
+    def __build_user_service(self, username, service='geocoding', quota=100,
+                             provider='heremaps', orgname=None,
+                             soft_limit=False, end_date=datetime.today()):
+        build_redis_user_config(self.redis_conn, username, service,
+                                quota=quota, provider=provider,
+                                soft_limit=soft_limit,
+                                end_date=end_date)
         if orgname:
-            build_redis_org_config(self.redis_conn, orgname,
-                                               quota=quota, end_date=end_date)
+            build_redis_org_config(self.redis_conn, orgname, service,
+                                   provider=provider, quota=quota,
+                                   end_date=end_date)
         geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
                                          username, orgname)
         return UserMetricsService(geocoder_config, self.redis_conn)

test/integration/test_data_observatory_functions.py

@@ -160,12 +160,12 @@ class TestDataObservatoryFunctions(TestCase):
             assert_equal(e.message[0], "The api_key must be provided")
 
     def test_if_obs_get_boundary_id_is_ok(self):
-        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;&api_key={0}".format(self.env_variables['api_key'])
+        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;&api_key={0}".format(self.env_variables['api_key'])
         result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
         assert_not_equal(result['boundary_id'], None)
 
     def test_if_obs_get_boundary_id_without_api_key_raise_error(self):
-        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2014') as boundary_id;"
+        query = "SELECT OBS_GetBoundaryId(CDB_LatLng(40.704512, -73.936669), 'us.census.tiger.census_tract', '2015') as boundary_id;"
         try:
             IntegrationTestHelper.execute_query(self.sql_api_url, query)
         except Exception as e:
@@ -290,3 +290,41 @@ class TestDataObservatoryFunctions(TestCase):
             IntegrationTestHelper.execute_query(self.sql_api_url, query)
         except Exception as e:
             assert_equal(e.message[0], "The api_key must be provided")
+
+    def test_if_obs_get_meta_is_ok(self):
+        params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
+        query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), {0}, 1, 1, 1000) as metadata LIMIT 1;&api_key={1}".format(params, self.env_variables['api_key'])
+        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        assert_not_equal(result['metadata'], None)
+
+    def test_if_obs_get_meta_without_api_key_raise_error(self):
+        params = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
+        query = "SELECT obs_getmeta(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), {0}, 1, 1, 1000) LIMIT 1;".format(params)
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")
+
+    def test_if_obs_get_data_is_ok(self):
+        params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\''
+        params_2 = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
+        query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0}, 1, 1, 1000)) LIMIT 1;&api_key={1}".format(params_1, self.env_variables['api_key'])
+        query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0})) LIMIT 1;&api_key={1}".format(params_2,self.env_variables['api_key'])
+        result_1 = IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
+        assert_not_equal(result_1['data_id'], None)
+        result_2 = IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
+        assert_not_equal(result_2['data_id'], None)
+
+    def test_if_obs_get_data_without_api_key_raise_error(self):
+        params_1 = '\'[{\"numer_id\": \"us.census.acs.B01003001\", \"geom_id\": \"us.census.tiger.county\"}]\'';
+        params_2 = '\'[{\"numer_id\": \"us.census.acs.B01003001\"}]\''
+        query_1 = "SELECT id as data_id FROM obs_getdata(ARRAY['36047'], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0}, 1, 1, 1000)) LIMIT 1;".format(params_1)
+        query_2 = "SELECT id as data_id FROM obs_getdata(ARRAY[(ST_SetSRID(ST_Point(-73.9, 40.7), 4326), 1)::geomval], obs_getmeta(st_setsrid(st_point(-73.9, 40.7), 4326), {0})) LIMIT 1;".format(params_2)
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query_1)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query_2)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")