Merge pull request #334 from CartoDB/fix-cannot-find-geometry

Add cartodb to the search path

NEWS.md

@@ -1,3 +1,28 @@
+November 29st, 2016
+===================
+* Version 0.20.0 of the server and version 0.12.0 of the python library
+  * Added integration with the new Mapzen isochrones functionality
+
+November 25st, 2016
+===================
+* Version 0.19.0 of the server, version 0.11.0 of the python library and version 0.13.0 of the client
+  * functions to check the quota, both server and client
+  * removed the no_params from the templates (this caused trouble, not needed anymore)
+  * bug fixes: observatory quota, quotas as integers, mapzen geocoder soft limit as bool
+
+November 21st, 2016
+===================
+* Version 0.18.1 of the server and version 0.12.1 of the client
+  * Add new fields to the obs_meta_geometry due to new changes introduced in the DO 1.1.2 release
+
+November 11st, 2016
+===================
+* Version 0.18.0 of the server and version 0.12.0 of the client
+  * Added obs_legacybuildermetada functions to grab the needed metadata in the builder while making a data enrichment analysis. Closes #286
+  * Added metadata functions that will be used in the future to gather the metadata for the new data enrichment UI. Closes #287
+  * Fixed integration test for street geocoding
+  * Makefile now has a new task to create a new release in the client part, as was in the server, using make release NEW_VERSION=x.x.x
+
 November 7st, 2016
 ==================
 * Version 0.17.0 of the server and version 0.10.0 of the python package

client/.gitignore

@@ -2,9 +2,10 @@ results/
 regression.diffs
 regression.out
 20_public_functions.sql
+25_exception_safe_private_functions.sql
 30_plproxy_functions.sql
 90_grant_execute.sql
 cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
-cdb_geocoder_client--0.3.0.sql
+cdb_geocoder_client--0.3.0.sql

client/Makefile

@@ -57,9 +57,11 @@ all: $(DATA)
 .PHONY: release
 release: $(EXTENSION).control $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
-	mv *.sql old_versions
+	git mv *.sql old_versions
 	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 

client/cdb_dataservices_client--0.14.0--0.14.1.sql

@@ -0,0 +1,9 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.1'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- This release introduces no changes other than the use of
+-- search path in the install and migration scripts

client/cdb_dataservices_client--0.14.1--0.14.0.sql

@@ -0,0 +1,9 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.14.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- This release introduces no changes other than the use of
+-- search path in the install and migration scripts

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.13.0'
+default_version = '0.14.1'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.14.0--0.13.0.sql

@@ -0,0 +1,52 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.13.0'" to load this file. \quit
+
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin0_polygon_exception_safe(country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_admin1_polygon_exception_safe(admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_namedplace_point_exception_safe(city_name text, admin1_name text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_polygon_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_postalcode_point_exception_safe(postal_code text, country_name text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_ipaddress_point_exception_safe(ip_address text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance_exception_safe(source geometry(Geometry, 4326), mode text, range integer[], options text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point_exception_safe(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_with_waypoints_exception_safe(waypoints geometry(Point, 4326)[], mode text, options text[], units text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom geometry(Geometry, 4326), time_span text, geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom geometry(Geometry, 4326), geometry_level text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure_exception_safe(geom Geometry, measure_id text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory_exception_safe(geom Geometry, category_id text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom Geometry, name text, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom Geometry, name text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation_exception_safe(geom Geometry, normalize text, boundary_id text, time_span text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom Geometry, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_exception_safe();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_enough_quota_exception_safe(service TEXT, input_size NUMERIC);
+

client/renderer/templates/25_exception_safe_private_functions.erb

@@ -0,0 +1,61 @@
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>)
+RETURNS <%= return_type %> AS $$
+DECLARE
+  <% if not multi_row %>ret <%= return_type %>;<% end %>
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  <% if multi_row %>
+    BEGIN
+      RETURN QUERY
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+    END;
+  <% elsif multi_field %>
+    BEGIN
+      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% else %>
+    BEGIN
+      SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
+      RETURN ret;
+    EXCEPTION
+      WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        RETURN ret;
+    END;
+  <% end %>
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

client/renderer/templates/90_grant_execute.erb

@@ -1 +1,2 @@
 GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;
+GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser;

client/sql/00_header.sql

@@ -1,3 +1,6 @@
 --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
 -- Complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;

client/test/expected/25_exception_safe_private_functions_test.out

@@ -0,0 +1,56 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+-- Use regular user role
+SET ROLE test_regular_user;
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+WARNING:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
+PL/pgSQL function _cdb_geocode_street_point_exception_safe(text,text,text,text) line 21 at SQL statement
+ _cdb_geocode_street_point_exception_safe 
+------------------------------------------
+ 
+(1 row)
+
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+WARNING:  cdb_dataservices_client._cdb_isodistance(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  PL/pgSQL function _cdb_isodistance_exception_safe(geometry,text,integer[],text[]) line 21 at RETURN QUERY
+ center | data_range | the_geom 
+--------+------------+----------
+(0 rows)
+
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+WARNING:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
+DETAIL:  SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)"
+PL/pgSQL function _cdb_route_point_to_point_exception_safe(geometry,geometry,text,text[],text) line 21 at SQL statement
+ shape | length | duration 
+-------+--------+----------
+       |        |         
+(1 row)
+

client/test/sql/25_exception_safe_private_functions_test.sql

@@ -0,0 +1,41 @@
+SET client_min_messages TO warning;
+SET search_path TO public,cartodb,cdb_dataservices_client;
+
+-- Mock the server functions to raise exceptions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+  RETURN NULL;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username text, orgname text, source geometry, mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF isoline AS $$
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers')
+RETURNS cdb_dataservices_client.simple_route AS $$
+DECLARE 
+  ret cdb_dataservices_client.simple_route;
+BEGIN
+  RAISE EXCEPTION 'Not enough quota or any other exception whatsoever.';
+
+  -- This code shall never be reached
+  SELECT NULL, 5.33, 100 INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
+
+
+-- Use regular user role
+SET ROLE test_regular_user;
+
+-- Exercise the exception safe and the proxied functions
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);

client/upgrade_downgrade_template.erb

@@ -2,4 +2,7 @@
 -- Complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
 
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
 -- HERE goes your code to upgrade/downgrade

doc/internal/exception_safe.md

@@ -0,0 +1,36 @@
+# Exception-Safe functions
+
+The public API dataservices functions emit exceptions in general when an error occurs
+or a limiting condition is met (e.g. quotas are exceeded).
+
+For each public function `func` we have a internal function named `_func_exception_safe` which
+acts as a wrapper to the public function, with the same signature, but captures
+exceptions generated during its execution (except those due to incomplete configuration or
+authentication issues) and returns NULL or empty set values in those cases.
+
+Please note these functions are considered **not public** and therefore their API (including which exceptions are wrapped and which ones are not) may change.
+
+Instead of raising an exception they raise warnings, hopefully containing the same information of the original exception.
+
+## Intended Use
+
+These functions are useful in cases when it is undesirable to rollback a transaction.
+Fo example if a table is geocoded with:
+
+```sql
+UPDATE table SET the_geom=cdb_geocode_street_point(user,NULL,address,city,NULL,country);
+```
+
+In case of the user geocoding quota being exhausted mid-process, the user could
+incur in external service expenses but any geocoded data would be lost due to the
+transaction rollback.
+
+We can avoid the problem using the corresponding exception-safe function:
+
+```sql
+UPDATE table SET the_geom=_cdb_geocode_street_point_exception_safe(user,NULL,address,city,NULL,country);
+```
+
+# Addition Information
+
+See https://github.com/CartoDB/dataservices-api/issues/314 for more information.

server/lib/python/cartodb_services/README.md

@@ -46,5 +46,7 @@ cd $(git rev-parse --show-toplevel)/test
 python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY
 ```
 
-## TODO
-- Move dependencies expressed in `requirements.txt` to `setup.py`
+## Versioning
+Once you're satisfied with your changes, it is time to bump the version number in the `setup.py`. A couple of rules:
+- **Backwards compatibility**: in general all changes shall be backwards compatible. Do not remove any code used from the server public `pl/python` functions or you'll run into problems when deploying.
+- **Semantic versioning**: we try to stick to [Semantic Versioning 2.0.0](http://semver.org/spec/v2.0.0.html)

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -4,6 +4,7 @@
 import json
 import requests
 
+from requests.adapters import HTTPAdapter
 from exceptions import *
 from cartodb_services.metrics import Traceable
 
@@ -17,6 +18,7 @@ class HereMapsGeocoder(Traceable):
     DEFAULT_GEN = 9
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ADDRESS_PARAMS = [
         'city',
@@ -88,7 +90,10 @@ class HereMapsGeocoder(Traceable):
             'gen': self.gen
         }
         request_params.update(params)
-        response = requests.get(self.host, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self.host, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 
 from exceptions import WrongParams
+from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 
 
@@ -13,6 +14,7 @@ class HereMapsRoutingIsoline(Traceable):
     ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -53,6 +55,9 @@ class HereMapsRoutingIsoline(Traceable):
                                                          data_range,
                                                          range_type,
                                                          parsed_options)
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
         response = requests.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)

server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenGeocoder(Traceable):
     BASE_URL = 'https://search.mapzen.com/v1/search'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     def __init__(self, app_key, logger, base_url=BASE_URL):
         self._app_key = app_key
@@ -27,7 +29,10 @@ class MapzenGeocoder(Traceable):
                                                          state_province,
                                                          country, search_type)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
             self.add_response_data(response, self._logger)
             if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 
@@ -12,6 +13,7 @@ class MapzenIsochrones:
     BASE_URL = 'https://matrix.mapzen.com/isochrone'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -23,12 +25,15 @@ class MapzenIsochrones:
         self._url = base_url
         self._logger = logger
 
-    @qps_retry
+    @qps_retry(qps=7)
     def isochrone(self, locations, costing, ranges):
         request_params = self._parse_request_params(locations, costing,
                                                     ranges)
         try:
-            response = requests.get(self._url, params=request_params,
+            # TODO Extract HTTP client wrapper
+            session = requests.Session()
+            session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+            response = session.get(self._url, params=request_params,
                                     timeout=(self.CONNECT_TIMEOUT,
                                             self.READ_TIMEOUT))
 

server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py

@@ -55,7 +55,7 @@ class QPSService:
 
     def retry(self, first_request_time, retry_count):
         elapsed = datetime.now() - first_request_time
-        if elapsed.microseconds  > (self._retry_timeout * 1000.0):
+        if elapsed.microseconds  > (self._retry_timeout * 1000.0 * 1000.0):
             raise TimeoutException()
 
         # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per

server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py

@@ -2,6 +2,7 @@ import requests
 import json
 import re
 
+from requests.adapters import HTTPAdapter
 from exceptions import WrongParams, MalformedResult, ServiceException
 from qps import qps_retry
 from cartodb_services.tools import Coordinate, PolyLine
@@ -14,6 +15,7 @@ class MapzenRouting(Traceable):
     PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
+    MAX_RETRIES=1
 
     ACCEPTED_MODES = {
         "walk": "pedestrian",
@@ -46,7 +48,10 @@ class MapzenRouting(Traceable):
                                                            mode_param,
                                                            units)
         request_params = self.__parse_request_parameters(json_request_params)
-        response = requests.get(self._url, params=request_params,
+        # TODO Extract HTTP client wrapper
+        session = requests.Session()
+        session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
+        response = session.get(self._url, params=request_params,
                                 timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
         self.add_response_data(response, self._logger)
         if response.status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -1,6 +1,15 @@
 from datetime import date, timedelta
 from dateutil.relativedelta import relativedelta
+from calendar import monthrange
 
+def last_day_of_month(year, month):
+    """last valid day of a month"""
+    return monthrange(year, month)[1]
+
+def latest_valid_date(year, month, day):
+    """latest date not later than the day specified"""
+    valid_day = min(day, last_day_of_month(year, month))
+    return date(year, month, valid_day)
 
 class UserMetricsService:
     """ Class to manage all the user info """
@@ -143,9 +152,9 @@ class UserMetricsService:
         today = date.today()
         if end_period_day > today.day:
             temp_date = today + relativedelta(months=-1)
-            date_from = date(temp_date.year, temp_date.month, end_period_day)
+            date_from = latest_valid_date(temp_date.year, temp_date.month, end_period_day)
         else:
-            date_from = date(today.year, today.month, end_period_day)
+            date_from = latest_valid_date(today.year, today.month, end_period_day)
 
         return date_from, today
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.12.0',
+    version='0.12.4',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_user_service.py

@@ -23,6 +23,10 @@ class TestUserService(TestCase):
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
 
+    def test_user_quota_for_a_month_shorter_than_end_day(self):
+        us = self.__build_user_service('test_user', end_date=date(2016,1,31))
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
+
     def test_org_used_quota_for_a_day(self):
         us = self.__build_user_service('test_user', orgname='test_org')
         increment_service_uses(self.redis_conn, 'test_user',
@@ -30,6 +34,10 @@ class TestUserService(TestCase):
                                            amount=400)
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
 
+    def test_org_quota_quota_for_a_month_shorter_than_end_day(self):
+        us = self.__build_user_service('test_user', orgname='test_org', end_date=date(2016,1,31))
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2016,2,10)) == 0
+
     def test_user_not_amount_in_used_quota_for_month_should_be_0(self):
         us = self.__build_user_service('test_user')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0