Merge branch 'development'

204 write zero padded keys

client/Makefile

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
 DATA =  $(NEW_EXTENSION_ARTIFACT) \
   $(OLD_VERSIONS) \
-  cdb_dataservices_client--0.5.0--0.6.0.sql \
+  cdb_dataservices_client--0.6.0--0.7.0.sql \
-  cdb_dataservices_client--0.6.0--0.5.0.sql
+  cdb_dataservices_client--0.7.0--0.6.0.sql
 
 
 REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))

client/cdb_dataservices_client--0.6.0--0.7.0.sql

@@ -0,0 +1,35 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.7.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_getmeasurebyid (geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL)
+RETURNS numeric AS $$
+DECLARE
+  ret numeric;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span) INTO ret;
+    RETURN ret;
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_getmeasurebyid (username text, organization_name text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL)
+RETURNS numeric AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.obs_getmeasurebyid (username, organization_name, geom_ref, measure_id, boundary_id, time_span);
+  
+$$ LANGUAGE plproxy;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) TO publicuser;

client/cdb_dataservices_client--0.7.0--0.6.0.sql

@@ -0,0 +1,5 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.6.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getmeasurebyid (text, text, text, text);

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.6.0'
+default_version = '0.7.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/renderer/interface.yaml

@@ -210,6 +210,14 @@
     - { name: boundary_id, type: text, default: 'NULL' }
     - { name: time_span, type: text, default: 'NULL'}
 
+- name: obs_getmeasurebyid
+  return_type: numeric
+  params:
+    - { name: geom_ref,  type: text }
+    - { name: measure_id, type: text }
+    - { name: boundary_id, type: text}
+    - { name: time_span, type: text, default: 'NULL'}
+
 - name: obs_getcategory
   return_type: text
   params:

client/test/expected/90_data_observatory_test.out

@@ -102,6 +102,16 @@ BEGIN
   RETURN ret;
 END;
 $$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL)
+RETURNS numeric AS $$
+DECLARE
+  ret numeric;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getmeasurebyid invoked with params (%, %, %, %, %, %)', username, orgname, geom_ref, measure_id, boundary_id, time_span;
+  SELECT 10923.093200390833950::numeric INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
 RETURNS text AS $$
 DECLARE
@@ -266,6 +276,15 @@ PL/pgSQL function obs_getmeasure(geometry,text,text,text,text) line 16 at SQL st
  10923.093200390833950
 (1 row)
 
+SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text);
+NOTICE:  cdb_dataservices_client._obs_getmeasurebyid(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getmeasurebyid invoked with params (test_user, <NULL>, 36047, us.census.acs.B01001001, us.census.tiger.county, <NULL>)
+CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_getmeasurebyid(username, orgname, geom_ref, measure_id, boundary_id, time_span)"
+PL/pgSQL function obs_getmeasurebyid(text,text,text,text) line 16 at SQL statement
+  obs_getmeasurebyid   
+-----------------------
+ 10923.093200390833950
+(1 row)
+
 SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text);
 NOTICE:  cdb_dataservices_client._obs_getcategory(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.obs_getcategory invoked with params (test_user, <NULL>, 0101000020E6100000548B8862F27B52C0DDD1FF722D5A4440, us.census.spielman_singleton_segments.X10, <NULL>, <NULL>)
 CONTEXT:  SQL statement "SELECT cdb_dataservices_client._obs_getcategory(username, orgname, geom, category_id, boundary_id, time_span)"

client/test/sql/90_data_observatory_test.sql

@@ -115,6 +115,17 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getmeasurebyid (username text, orgname text, geom_ref text, measure_id text, boundary_id text, time_span text DEFAULT NULL)
+RETURNS numeric AS $$
+DECLARE
+  ret numeric;
+BEGIN
+  RAISE NOTICE 'cdb_dataservices_server.obs_getmeasurebyid invoked with params (%, %, %, %, %, %)', username, orgname, geom_ref, measure_id, boundary_id, time_span;
+  SELECT 10923.093200390833950::numeric INTO ret;
+  RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql';
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_getcategory (username text, orgname text, geom Geometry, category_id text, boundary_id text DEFAULT NULL, time_span text DEFAULT NULL)
 RETURNS text AS $$
 DECLARE
@@ -196,6 +207,7 @@ SELECT obs_getboundariesbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704
 SELECT obs_getpointsbygeometry(ST_MakeEnvelope(-73.9452409744, 40.6988851644, -73.9280319214, 40.7101254524, 4326), 'us.census.tiger.census_tract'::text);
 SELECT obs_getpointsbypointandradius(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 500::numeric, 'us.census.tiger.census_tract'::text);
 SELECT obs_getmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.acs.B01001001'::text);
+SELECT obs_getmeasurebyid('36047'::text, 'us.census.acs.B01001001'::text, 'us.census.tiger.county'::text);
 SELECT obs_getcategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'us.census.spielman_singleton_segments.X10'::text);
 SELECT obs_getuscensusmeasure(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'male population'::text);
 SELECT obs_getuscensuscategory(ST_SetSRID(ST_Point(-73.936669 , 40.704512), 4326), 'Spielman-Singleton Segments: 10 Clusters'::text);

doc/routing_functions.md

@@ -2,7 +2,7 @@
 
 Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CartoDB are available by using the available functions in the Data Services API.
 
-### cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)
+## cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)
 
 Returns a route from origin to destination.
 
@@ -38,7 +38,7 @@ INSERT INTO <TABLE> (duration, length, the_geom) SELECT duration, length, shape
 UPDATE <TABLE> SET the_geom = (SELECT shape FROM cdb_route_point_to_point('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]))
 ```
 
-### cdb_route_with_waypoints(_waypoints geometry(Point)[], mode text, [options text[], units text]_)
+## cdb_route_with_waypoints(_waypoints geometry(Point)[], mode text, [options text[], units text]_)
 
 Returns a route that goes from origin to destination and whose path travels through the defined locations.
 

server/extension/Makefile

@@ -2,39 +2,47 @@
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
 EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
-
 # The new version to be generated from templates
+SED = sed
+ERB = erb
+REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
 NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-OLD_VERSIONS = $(wildcard old_versions/*.sql)
+
+REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
+TEST_DIR = test/
+REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
 
 # DATA is a special variable used by postgres build infrastructure
 # These are the files to be installed in the server shared dir,
 # for installation from scratch, upgrades and downgrades.
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
-DATA =  $(NEW_EXTENSION_ARTIFACT) \
+DATA = $(EXTENSION)--*.sql
-  $(OLD_VERSIONS) \
+OLD_VERSIONS = $(wildcard old_versions/*.sql)
-  cdb_dataservices_server--0.8.0--0.9.0.sql \
+SOURCES_DATA_DIR = sql/
-  cdb_dataservices_server--0.9.0--0.8.0.sql
+SOURCES_DATA = $(wildcard sql/*.sql)
-
-REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
-TEST_DIR = test/
-REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
 
 # postgres build stuff
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-SOURCES_DATA_DIR = sql/
-
-SOURCES_DATA = $(wildcard sql/*.sql)
-
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
+.PHONY: all
 all: $(DATA)
 
+.PHONY: release
+release: $(EXTENSION).control $(SOURCES_DATA)
+	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
+	mv *.sql old_versions
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
+	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
+
 # Only meant for development time, do not use once a version is released
+.PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)

server/extension/cdb_dataservices_server--0.10.0--0.9.0.sql

@@ -0,0 +1,5 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.9.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetMeasureById(text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_GetMeasureById(text, text, text, text, text, text);

server/extension/cdb_dataservices_server--0.9.0--0.10.0.sql

@@ -0,0 +1,54 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.10.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
+  username TEXT,
+  orgname TEXT,
+  geom_ref TEXT,
+  measure_id TEXT,
+  boundary_id TEXT,
+  time_span TEXT DEFAULT NULL)
+RETURNS NUMERIC AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
+  username TEXT,
+  orgname TEXT,
+  geom_ref TEXT,
+  measure_id TEXT,
+  boundary_id TEXT,
+  time_span TEXT DEFAULT NULL)
+RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import QuotaService
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
+      result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['measure']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.9.0'
+default_version = '0.10.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/110_data_observatory_augmentation.sql

@@ -476,3 +476,54 @@ RETURNS NUMERIC AS $$
   finally:
       quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
+  username TEXT,
+  orgname TEXT,
+  geom_ref TEXT,
+  measure_id TEXT,
+  boundary_id TEXT,
+  time_span TEXT DEFAULT NULL)
+RETURNS NUMERIC AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.OBS_GetMeasureById(geom_ref, measure_id, boundary_id, time_span);
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
+  username TEXT,
+  orgname TEXT,
+  geom_ref TEXT,
+  measure_id TEXT,
+  boundary_id TEXT,
+  time_span TEXT DEFAULT NULL)
+RETURNS NUMERIC AS $$
+  from cartodb_services.metrics import QuotaService
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+
+  quota_service = QuotaService(user_obs_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+      obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
+      result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
+      if result:
+        quota_service.increment_success_service_use()
+        return result[0]['measure']
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+  except BaseException as e:
+      import sys, traceback
+      type_, value_, traceback_ = sys.exc_info()
+      quota_service.increment_failed_service_use()
+      error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e)
+      plpy.notice(traceback.format_tb(traceback_))
+      plpy.error(error_msg)
+  finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/test/expected/100_data_observatory_test.out

@@ -31,6 +31,17 @@ SELECT exists(SELECT *
  t
 (1 row)
 
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getmeasurebyid'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, text');
+ exists 
+--------
+ t
+(1 row)
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/test/sql/100_data_observatory_test.sql

@@ -19,6 +19,13 @@ SELECT exists(SELECT *
               AND proname = 'obs_getmeasure'
               AND oidvectortypes(p.proargtypes)  = 'text, text, geometry, text, text, text, text');
 
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'obs_getmeasurebyid'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, text');
+
 SELECT exists(SELECT *
               FROM pg_proc p
               INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)

server/extension/upgrade_downgrade_template.erb

@@ -0,0 +1,5 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade

server/lib/python/cartodb_services/README.md

@@ -0,0 +1,50 @@
+# CartoDB dataservices API python module
+
+This directory contains the python library used by the server side of CartoDB LDS (Location Data Services).
+
+It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests.
+
+On the other hand, it is pretty independent from the client, as long as the signatures of the public pl/python functions match.
+
+## Dependencies
+See the [[`requirements.txt`]] or better the Basically:
+- pip
+- redis and hiredis
+- dateutil
+- googlemaps
+- request
+
+## Installation
+Install the requirements:
+```shell
+sudo pip install -r requirements.txt
+```
+
+Install the library:
+```shell
+sudo pip install .
+```
+
+NOTE: a system installation is required at present because the library is meant to be used from postgres pl/python, which runs an embedded python interpreter.
+
+
+## Running the unit tests
+Just run `nosetests`
+```shell
+$ nosetests
+.................................................
+----------------------------------------------------------------------
+Ran 49 tests in 0.131s
+
+OK
+```
+
+## Running the integration tests
+See the [[../../../../test/README.md]]. Basically, move to the `/test` directory at the top level of this repo and execute the `run_tests.py` script:
+```sh
+cd $(git rev-parse --show-toplevel)/test
+python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY
+```
+
+## TODO
+- Move dependencies expressed in `requirements.txt` to `setup.py`

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -8,6 +8,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
         self._user_geocoder_config = user_geocoder_config
@@ -85,6 +86,11 @@ class UserMetricsService:
                                                      service, metric, date)
             score = self._redis_connection.zscore(redis_prefix, date.day)
             aggregated_metric += score if score else 0
+            zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
+            if str(date.day) != zero_padded_day:
+                score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
+                aggregated_metric += score if score else 0
+
         return aggregated_metric
 
     # Private functions
@@ -92,12 +98,16 @@ class UserMetricsService:
     def __increment_user_uses(self, service_type, metric, date, amount):
         redis_prefix = self.__parse_redis_prefix("user", self._username,
                                                  service_type, metric, date)
-        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+        self._redis_connection.zincrby(redis_prefix,
+                                       date.strftime(self.DAY_OF_MONTH_ZERO_PADDED),
+                                       amount)
 
     def __increment_organization_uses(self, service_type, metric, date, amount):
         redis_prefix = self.__parse_redis_prefix("org", self._orgname,
                                                  service_type, metric, date)
-        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+        self._redis_connection.zincrby(redis_prefix,
+                                       date.strftime(self.DAY_OF_MONTH_ZERO_PADDED),
+                                       amount)
 
     def __parse_redis_prefix(self, prefix, entity_name, service_type, metric,
                              date):

server/lib/python/cartodb_services/requirements.txt

@@ -1,6 +1,5 @@
 redis==2.10.5
 hiredis==0.1.5
-# Dependency with incsv in the import
 python-dateutil==2.2
 googlemaps==2.4.2
 # Dependency for googlemaps package
@@ -11,3 +10,4 @@ mock==1.3.0
 mockredispy==2.9.0.11
 nose==1.3.7
 requests-mock==0.7.0
+freezegun==0.3.7

server/lib/python/cartodb_services/setup.py

@@ -10,11 +10,11 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.6.2',
+    version='0.6.4',
 
     description='CartoDB Services API Python Library',
 
-    url='https://github.com/CartoDB/geocoder-api',
+    url='https://github.com/CartoDB/dataservices-api',
 
     author='Data Services Team - CartoDB',
     author_email='dataservices@cartodb.com',

server/lib/python/cartodb_services/test/test_user_service.py

@@ -7,6 +7,7 @@ from unittest import TestCase
 from mock import Mock
 from nose.tools import assert_raises
 from datetime import timedelta
+from freezegun import freeze_time
 
 
 class TestUserService(TestCase):
@@ -75,9 +76,65 @@ class TestUserService(TestCase):
         us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
+    @freeze_time("2015-06-01")
+    def test_should_account_for_zero_paddded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_account_for_wrongly_stored_non_padded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_sum_amounts_from_both_key_formats(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506',  '1', 400)
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700
+
+    @freeze_time("2015-06-15")
+    def test_should_not_request_redis_twice_when_unneeded(self):
+        class MockRedisWithCounter(MockRedis):
+            def __init__(self):
+                super(MockRedisWithCounter, self).__init__()
+                self._zscore_counter = 0
+            def zscore(self, *args):
+                print args
+                self._zscore_counter += 1
+                return super(MockRedisWithCounter, self).zscore(*args)
+            def zscore_counter(self):
+                return self._zscore_counter
+        self.redis_conn = MockRedisWithCounter()
+        us = self.__build_user_service('test_user', end_date=date.today())
+        us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15))
+
+        #('user:test_user:geocoder_here:success_responses:201506', 15)
+        #('user:test_user:geocoder_here:empty_responses:201506', 15)
+        #('user:test_user:geocoder_cache:success_responses:201506', 15)
+        assert self.redis_conn.zscore_counter() == 3
+
+    def test_should_write_zero_padded_dates(self):
+        us = self.__build_user_service('test_user')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
+                                 date=date(2015,6,1))
+        assert self.redis_conn.zscore('user:test_user:geocoder_here:success_responses:201506', '01') == 1
+        assert self.redis_conn.zscore('user:test_user:geocoder_here:success_responses:201506',  '1') == None
+
+    def test_orgs_should_write_zero_padded_dates(self):
+        us = self.__build_user_service('test_user', orgname='test_org')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
+                                 amount=400,
+                                 date=date(2015,6,1))
+        assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506', '01') == 400
+        assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506',  '1') == None
+
+
     def __build_user_service(self, username, quota=100, service='heremaps',
                              orgname=None, soft_limit=False,
-                             end_date=datetime.today()):
+                             end_date=date.today()):
         test_helper.build_redis_user_config(self.redis_conn, username,
                                             quota=quota, service=service,
                                             soft_limit=soft_limit,

test/integration/test_data_observatory_functions.py

@@ -57,6 +57,19 @@ class TestDataObservatoryFunctions(TestCase):
         except Exception as e:
             assert_equal(e.message[0], "The api_key must be provided")
 
+    def test_if_get_measure_by_id_ok(self):
+        query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure;&api_key={0}".format(self.env_variables['api_key'])
+        result = IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        assert_not_equal(result['measure'], None)
+        assert_equal(result['measure'], 3241)
+
+    def test_if_get_measure_by_id_without_api_key_raise_error(self):
+        query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure"
+        try:
+            IntegrationTestHelper.execute_query(self.sql_api_url, query)
+        except Exception as e:
+            assert_equal(e.message[0], "The api_key must be provided")
+
     def test_if_get_category_is_ok(self):
         query = "SELECT OBS_GetCategory(CDB_LatLng(40.704512, -73.936669), 'us.census.spielman_singleton_segments.X10', 'us.census.tiger.census_tract', '2010 - 2014') as category;&api_key={0}".format(self.env_variables['api_key'])
         result = IntegrationTestHelper.execute_query(self.sql_api_url, query)

test/integration/test_routing_functions.py

@@ -35,7 +35,7 @@ class TestRoutingFunctions(TestCase):
 
     def test_if_select_with_routing_with_waypoints_is_ok(self):
         query = "SELECT duration, length, shape as the_geom " \
-                "FROM cdb_route_with_waypoints('Array['POINT(-3.7109 40.4234)'::GEOMETRY, "\
+                "FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::GEOMETRY, "\
                 "'POINT(-3.7059 40.4203)'::geometry, 'POINT(-3.7046 40.4180)'::geometry]" \
                 "::geometry[], 'car', " \
                 "ARRAY['mode_type=shortest']::text[])&api_key={0}".format(
@@ -45,7 +45,7 @@ class TestRoutingFunctions(TestCase):
 
     def test_if_select_with_routing_with_waypoints_without_api_key_raise_error(self):
         query = "SELECT duration, length, shape as the_geom " \
-                "FROM cdb_route_with_waypoints('Array['POINT(-3.7109 40.4234)'::GEOMETRY, "\
+                "FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::GEOMETRY, "\
                 "'POINT(-3.7059 40.4203)'::geometry, 'POINT(-3.7046 40.4180)'::geometry]" \
                 "::geometry[], 'car', " \
                 "ARRAY['mode_type=shortest']::text[])&api_key={0}"