Merge branch 'development'

204 write zero padded keys

server/extension/Makefile

@@ -2,39 +2,47 @@
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
 EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
-
 # The new version to be generated from templates
+SED = sed
+ERB = erb
+REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
 NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-OLD_VERSIONS = $(wildcard old_versions/*.sql)
+
+REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
+TEST_DIR = test/
+REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
 
 # DATA is a special variable used by postgres build infrastructure
 # These are the files to be installed in the server shared dir,
 # for installation from scratch, upgrades and downgrades.
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
-DATA =  $(NEW_EXTENSION_ARTIFACT) \
-  $(OLD_VERSIONS) \
-  cdb_dataservices_server--0.9.0--0.10.0.sql \
-  cdb_dataservices_server--0.10.0--0.9.0.sql
-
-REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
-TEST_DIR = test/
-REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
+DATA = $(EXTENSION)--*.sql
+OLD_VERSIONS = $(wildcard old_versions/*.sql)
+SOURCES_DATA_DIR = sql/
+SOURCES_DATA = $(wildcard sql/*.sql)
 
 # postgres build stuff
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-SOURCES_DATA_DIR = sql/
-
-SOURCES_DATA = $(wildcard sql/*.sql)
-
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
+.PHONY: all
 all: $(DATA)
 
+.PHONY: release
+release: $(EXTENSION).control $(SOURCES_DATA)
+	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
+	mv *.sql old_versions
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control
+	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
+	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
+
 # Only meant for development time, do not use once a version is released
+.PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)

server/extension/upgrade_downgrade_template.erb

@@ -0,0 +1,5 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade

server/lib/python/cartodb_services/README.md

@@ -0,0 +1,50 @@
+# CartoDB dataservices API python module
+
+This directory contains the python library used by the server side of CartoDB LDS (Location Data Services).
+
+It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests.
+
+On the other hand, it is pretty independent from the client, as long as the signatures of the public pl/python functions match.
+
+## Dependencies
+See the [[`requirements.txt`]] or better the Basically:
+- pip
+- redis and hiredis
+- dateutil
+- googlemaps
+- request
+
+## Installation
+Install the requirements:
+```shell
+sudo pip install -r requirements.txt
+```
+
+Install the library:
+```shell
+sudo pip install .
+```
+
+NOTE: a system installation is required at present because the library is meant to be used from postgres pl/python, which runs an embedded python interpreter.
+
+
+## Running the unit tests
+Just run `nosetests`
+```shell
+$ nosetests
+.................................................
+----------------------------------------------------------------------
+Ran 49 tests in 0.131s
+
+OK
+```
+
+## Running the integration tests
+See the [[../../../../test/README.md]]. Basically, move to the `/test` directory at the top level of this repo and execute the `run_tests.py` script:
+```sh
+cd $(git rev-parse --show-toplevel)/test
+python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY
+```
+
+## TODO
+- Move dependencies expressed in `requirements.txt` to `setup.py`

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -8,6 +8,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_NOKIA = 'geocoder_here'
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
+    DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
         self._user_geocoder_config = user_geocoder_config
@@ -85,6 +86,11 @@ class UserMetricsService:
                                                      service, metric, date)
             score = self._redis_connection.zscore(redis_prefix, date.day)
             aggregated_metric += score if score else 0
+            zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
+            if str(date.day) != zero_padded_day:
+                score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
+                aggregated_metric += score if score else 0
+
         return aggregated_metric
 
     # Private functions
@@ -92,12 +98,16 @@ class UserMetricsService:
     def __increment_user_uses(self, service_type, metric, date, amount):
         redis_prefix = self.__parse_redis_prefix("user", self._username,
                                                  service_type, metric, date)
-        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+        self._redis_connection.zincrby(redis_prefix,
+                                       date.strftime(self.DAY_OF_MONTH_ZERO_PADDED),
+                                       amount)
 
     def __increment_organization_uses(self, service_type, metric, date, amount):
         redis_prefix = self.__parse_redis_prefix("org", self._orgname,
                                                  service_type, metric, date)
-        self._redis_connection.zincrby(redis_prefix, date.day, amount)
+        self._redis_connection.zincrby(redis_prefix,
+                                       date.strftime(self.DAY_OF_MONTH_ZERO_PADDED),
+                                       amount)
 
     def __parse_redis_prefix(self, prefix, entity_name, service_type, metric,
                              date):

server/lib/python/cartodb_services/requirements.txt

@@ -1,6 +1,5 @@
 redis==2.10.5
 hiredis==0.1.5
-# Dependency with incsv in the import
 python-dateutil==2.2
 googlemaps==2.4.2
 # Dependency for googlemaps package
@@ -11,3 +10,4 @@ mock==1.3.0
 mockredispy==2.9.0.11
 nose==1.3.7
 requests-mock==0.7.0
+freezegun==0.3.7

server/lib/python/cartodb_services/setup.py

@@ -10,11 +10,11 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.6.2',
+    version='0.6.4',
 
     description='CartoDB Services API Python Library',
 
-    url='https://github.com/CartoDB/geocoder-api',
+    url='https://github.com/CartoDB/dataservices-api',
 
     author='Data Services Team - CartoDB',
     author_email='dataservices@cartodb.com',

server/lib/python/cartodb_services/test/test_user_service.py

@@ -7,6 +7,7 @@ from unittest import TestCase
 from mock import Mock
 from nose.tools import assert_raises
 from datetime import timedelta
+from freezegun import freeze_time
 
 
 class TestUserService(TestCase):
@@ -75,9 +76,65 @@ class TestUserService(TestCase):
         us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses')
         assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2
 
+    @freeze_time("2015-06-01")
+    def test_should_account_for_zero_paddded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_account_for_wrongly_stored_non_padded_keys(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
+
+    @freeze_time("2015-06-01")
+    def test_should_sum_amounts_from_both_key_formats(self):
+        us = self.__build_user_service('test_user')
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506',  '1', 400)
+        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300)
+        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700
+
+    @freeze_time("2015-06-15")
+    def test_should_not_request_redis_twice_when_unneeded(self):
+        class MockRedisWithCounter(MockRedis):
+            def __init__(self):
+                super(MockRedisWithCounter, self).__init__()
+                self._zscore_counter = 0
+            def zscore(self, *args):
+                print args
+                self._zscore_counter += 1
+                return super(MockRedisWithCounter, self).zscore(*args)
+            def zscore_counter(self):
+                return self._zscore_counter
+        self.redis_conn = MockRedisWithCounter()
+        us = self.__build_user_service('test_user', end_date=date.today())
+        us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15))
+
+        #('user:test_user:geocoder_here:success_responses:201506', 15)
+        #('user:test_user:geocoder_here:empty_responses:201506', 15)
+        #('user:test_user:geocoder_cache:success_responses:201506', 15)
+        assert self.redis_conn.zscore_counter() == 3
+
+    def test_should_write_zero_padded_dates(self):
+        us = self.__build_user_service('test_user')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
+                                 date=date(2015,6,1))
+        assert self.redis_conn.zscore('user:test_user:geocoder_here:success_responses:201506', '01') == 1
+        assert self.redis_conn.zscore('user:test_user:geocoder_here:success_responses:201506',  '1') == None
+
+    def test_orgs_should_write_zero_padded_dates(self):
+        us = self.__build_user_service('test_user', orgname='test_org')
+        us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses',
+                                 amount=400,
+                                 date=date(2015,6,1))
+        assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506', '01') == 400
+        assert self.redis_conn.zscore('org:test_org:geocoder_here:success_responses:201506',  '1') == None
+
+
     def __build_user_service(self, username, quota=100, service='heremaps',
                              orgname=None, soft_limit=False,
-                             end_date=datetime.today()):
+                             end_date=date.today()):
         test_helper.build_redis_user_config(self.redis_conn, username,
                                             quota=quota, service=service,
                                             soft_limit=soft_limit,

test/integration/test_data_observatory_functions.py

@@ -63,7 +63,7 @@ class TestDataObservatoryFunctions(TestCase):
         assert_not_equal(result['measure'], None)
         assert_equal(result['measure'], 3241)
 
-    def test_if_get_measure_without_api_key_raise_error(self):
+    def test_if_get_measure_by_id_without_api_key_raise_error(self):
         query = "SELECT OBS_GetMeasureById('36047048500', 'us.census.acs.B01003001', 'us.census.tiger.census_tract', '2010 - 2014') as measure"
         try:
             IntegrationTestHelper.execute_query(self.sql_api_url, query)