From 9ad55a4d532dc98f257206ae82b5d06f4203a4d4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 10 Jun 2016 13:03:09 +0200 Subject: [PATCH 01/10] Update url of pip package --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index f5584fb..1d8d1d9 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -14,7 +14,7 @@ setup( description='CartoDB Services API Python Library', - url='https://github.com/CartoDB/geocoder-api', + url='https://github.com/CartoDB/dataservices-api', author='Data Services Team - CartoDB', author_email='dataservices@cartodb.com', From c7bcbddaa9a08adf69c0c68c5d2cdf479979b9d9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 11:16:44 +0200 Subject: [PATCH 02/10] Delete copy/pasted comment from requirements.txt #201 --- server/lib/python/cartodb_services/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/server/lib/python/cartodb_services/requirements.txt b/server/lib/python/cartodb_services/requirements.txt index 17e41c8..6a106af 100644 --- a/server/lib/python/cartodb_services/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -1,6 +1,5 @@ redis==2.10.5 hiredis==0.1.5 -# Dependency with incsv in the import python-dateutil==2.2 googlemaps==2.4.2 # Dependency for googlemaps package From 901f5d7b8b1c835669a5aaca96bae59266fc426c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 11:19:05 +0200 Subject: [PATCH 03/10] Add a README.md file for dummies like me #201 --- server/lib/python/cartodb_services/README.md | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 server/lib/python/cartodb_services/README.md diff --git a/server/lib/python/cartodb_services/README.md b/server/lib/python/cartodb_services/README.md new file mode 100644 index 0000000..9df0571 --- /dev/null +++ b/server/lib/python/cartodb_services/README.md @@ -0,0 +1,46 @@ +# CartoDB dataservices API python module + +This directory contains the python library used by the server side of CartoDB LDS (Location Data Services). + +It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests. + +On the other hand, it is pretty independent from the client, as long as the signatures of the public pl/python functions match. + +## Dependencies +See the [[`requirements.txt`]] or better the Basically: +- pip +- redis and hiredis +- dateutil +- googlemaps +- request + +## Installation +Install the requirements: +```shell +sudo pip install -r requirements.txt +``` + +Install the library: +```shell +sudo pip install . +``` + +NOTE: a system installation is required at present because the library is meant to be used from postgres pl/python, which runs an embedded python interpreter. + + +## Running the unit tests +Just run `nosetests` +```shell +$ nosetests +................................................. +---------------------------------------------------------------------- +Ran 49 tests in 0.131s + +OK +``` + +## Running the integration tests +TBD + +## TODO +- Move dependencies expressed in `requirements.txt` to `setup.py` From 47962c9ccbbfe588570e62d70cf120b9f98fc4df Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 11:33:11 +0200 Subject: [PATCH 04/10] Stub tests to be implemented (WIP) #201 --- .../cartodb_services/test/test_user_service.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index b173d31..00572bf 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -7,6 +7,7 @@ from unittest import TestCase from mock import Mock from nose.tools import assert_raises from datetime import timedelta +import nose #TODO remove class TestUserService(TestCase): @@ -75,6 +76,19 @@ class TestUserService(TestCase): us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + def test_should_account_for_zero_paddded_keys(self): + raise nose.SkipTest('not implemented yet') + + def test_should_account_for_wrongly_stored_non_padded_keys(self): + raise nose.SkipTest('not implemented yet') + + def test_should_sum_amounts_from_both_key_formats(self): + raise nose.SkipTest('not implemented yet') + + def test_should_not_request_redis_twice_when_unneeded(self): + raise nose.SkipTest('not implemented yet') + + def __build_user_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, end_date=datetime.today()): From a114f857fde62ad8cb73f79fc26177e4dee91432 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 12:21:06 +0200 Subject: [PATCH 05/10] Add the first test that makes sense #201 --- server/lib/python/cartodb_services/test/test_user_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 00572bf..54c7838 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -80,7 +80,9 @@ class TestUserService(TestCase): raise nose.SkipTest('not implemented yet') def test_should_account_for_wrongly_stored_non_padded_keys(self): - raise nose.SkipTest('not implemented yet') + us = self.__build_user_service('test_user', end_date = date(2016, 6, 1)) + self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201606', '1', 400) + assert us.used_quota(self.NOKIA_GEOCODER, date(2016, 6,1)) == 400 def test_should_sum_amounts_from_both_key_formats(self): raise nose.SkipTest('not implemented yet') From a2863d713563a9cbebebe1fd8d5caf845dd06693 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 12:57:00 +0200 Subject: [PATCH 06/10] Allow for the manipulation of datetime.date.today() #201 --- server/lib/python/cartodb_services/requirements.txt | 1 + .../lib/python/cartodb_services/test/test_user_service.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/requirements.txt b/server/lib/python/cartodb_services/requirements.txt index 6a106af..61426a1 100644 --- a/server/lib/python/cartodb_services/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -10,3 +10,4 @@ mock==1.3.0 mockredispy==2.9.0.11 nose==1.3.7 requests-mock==0.7.0 +freezegun==0.3.7 diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 54c7838..a77622d 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -8,6 +8,7 @@ from mock import Mock from nose.tools import assert_raises from datetime import timedelta import nose #TODO remove +from freezegun import freeze_time class TestUserService(TestCase): @@ -79,10 +80,11 @@ class TestUserService(TestCase): def test_should_account_for_zero_paddded_keys(self): raise nose.SkipTest('not implemented yet') + @freeze_time("2015-06-01") def test_should_account_for_wrongly_stored_non_padded_keys(self): - us = self.__build_user_service('test_user', end_date = date(2016, 6, 1)) - self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201606', '1', 400) - assert us.used_quota(self.NOKIA_GEOCODER, date(2016, 6,1)) == 400 + us = self.__build_user_service('test_user') + self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400) + assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400 def test_should_sum_amounts_from_both_key_formats(self): raise nose.SkipTest('not implemented yet') From b0bb60736c4c809efe196168871f866f5c90be2b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 13:07:21 +0200 Subject: [PATCH 07/10] Implement test and fix for that case #201 Add test_should_account_for_zero_paddded_keys plus its corresponding fix for it. --- .../python/cartodb_services/cartodb_services/metrics/user.py | 5 +++++ server/lib/python/cartodb_services/test/test_user_service.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index d154e0c..90c9698 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -85,6 +85,11 @@ class UserMetricsService: service, metric, date) score = self._redis_connection.zscore(redis_prefix, date.day) aggregated_metric += score if score else 0 + zero_padded_day = date.strftime('%d') + if str(date.day) != zero_padded_day: + score = self._redis_connection.zscore(redis_prefix, zero_padded_day) + aggregated_metric += score if score else 0 + return aggregated_metric # Private functions diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index a77622d..2f1a231 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -77,8 +77,11 @@ class TestUserService(TestCase): us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + @freeze_time("2015-06-01") def test_should_account_for_zero_paddded_keys(self): - raise nose.SkipTest('not implemented yet') + us = self.__build_user_service('test_user') + self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400) + assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400 @freeze_time("2015-06-01") def test_should_account_for_wrongly_stored_non_padded_keys(self): From f9e9617d6f51928a08f9a25c3efd60381e66c852 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 13:10:07 +0200 Subject: [PATCH 08/10] test_should_sum_amounts_from_both_key_formats #201 --- .../lib/python/cartodb_services/test/test_user_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 2f1a231..fdb7895 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -89,8 +89,12 @@ class TestUserService(TestCase): self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400) assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400 + @freeze_time("2015-06-01") def test_should_sum_amounts_from_both_key_formats(self): - raise nose.SkipTest('not implemented yet') + us = self.__build_user_service('test_user') + self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400) + self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300) + assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700 def test_should_not_request_redis_twice_when_unneeded(self): raise nose.SkipTest('not implemented yet') From 25d42a75e448dcc1309c34bc60a1f85a7a96adf4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 15:45:56 +0200 Subject: [PATCH 09/10] test_should_not_request_redis_twice_when_unneeded #201 --- .../test/test_user_service.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index fdb7895..fc0b6c6 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -7,7 +7,6 @@ from unittest import TestCase from mock import Mock from nose.tools import assert_raises from datetime import timedelta -import nose #TODO remove from freezegun import freeze_time @@ -96,13 +95,31 @@ class TestUserService(TestCase): self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300) assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700 + @freeze_time("2015-06-15") def test_should_not_request_redis_twice_when_unneeded(self): - raise nose.SkipTest('not implemented yet') + class MockRedisWithCounter(MockRedis): + def __init__(self): + super(MockRedisWithCounter, self).__init__() + self._zscore_counter = 0 + def zscore(self, *args): + print args + self._zscore_counter += 1 + return super(MockRedisWithCounter, self).zscore(*args) + def zscore_counter(self): + return self._zscore_counter + self.redis_conn = MockRedisWithCounter() + us = self.__build_user_service('test_user', end_date=date.today()) + us.used_quota(self.NOKIA_GEOCODER, date(2015, 6, 15)) + + #('user:test_user:geocoder_here:success_responses:201506', 15) + #('user:test_user:geocoder_here:empty_responses:201506', 15) + #('user:test_user:geocoder_cache:success_responses:201506', 15) + assert self.redis_conn.zscore_counter() == 3 def __build_user_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, - end_date=datetime.today()): + end_date=date.today()): test_helper.build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, From 41b9db4144cfdcfa5261ac33881168d3c38fb6e7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 13 Jun 2016 17:21:15 +0200 Subject: [PATCH 10/10] Update python lib to 0.6.3 #201 --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 1d8d1d9..6b46234 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.6.2', + version='0.6.3', description='CartoDB Services API Python Library',