From 797a2dbd2477d0be11f753cf0b500f53a94cfcd3 Mon Sep 17 00:00:00 2001 From: Antonio Date: Fri, 11 Aug 2017 10:21:10 +0200 Subject: [PATCH 1/4] Updated the User database configuration section --- README.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 25ef180..ecffea5 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Steps to deploy a new Data Services API version : ``` - Give permission to execute and select to the `dataservices_user` user: ``` - psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory; COMMIT" -e + psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory version 'dev'; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO dataservices_user; COMMIT" -e @@ -254,14 +254,16 @@ jsonb_set( ``` ### User database configuration +#### Option 1 + User (client) databases need also some configuration so that the client extension can access the server: -#### Users/Organizations +##### Users/Organizations ```sql SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); ``` -#### Dataservices server +##### Dataservices server The `geocoder_server_config` (the name is not accurate for historical reasons) entry points to the dataservices server DB (you can use a specific database for the server or your same user's): @@ -272,10 +274,31 @@ SELECT CDB_Conf_SetConf( '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}' ); ``` -#### Search path +##### Search path The search path must be configured in order to be able to execute the functions without using the schema: ```sql ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; ``` + +#### Option 2 + +Add the following entry to the `geocoder` entry of the `cartodb/config/app_config.yml` file: +``` +api: + host: 'localhost' + port: '5432' + user: 'dataservices_user' + dbname: 'dataservices_db' +``` + +In the `cartodb/config/app_config.yml` file, enable the desired dataservices: +``` +enabled: + geocoder_internal: false + hires_geocoder: false + isolines: false + routing: false + data_observatory: true +``` From e30b081cbc3d321338195a5468fa5e5caae103b4 Mon Sep 17 00:00:00 2001 From: Antonio Date: Wed, 16 Aug 2017 11:11:33 +0200 Subject: [PATCH 2/4] Added suggestions from CR --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ecffea5..1db1d33 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Steps to deploy a new Data Services API version : ``` - Give permission to execute and select to the `dataservices_user` user: ``` - psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory version 'dev'; COMMIT" -e + psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory VERSION 'dev'; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO dataservices_user; COMMIT" -e @@ -254,7 +254,7 @@ jsonb_set( ``` ### User database configuration -#### Option 1 +#### Option 1 (manually) User (client) databases need also some configuration so that the client extension can access the server: ##### Users/Organizations @@ -282,7 +282,7 @@ The search path must be configured in order to be able to execute the functions ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; ``` -#### Option 2 +#### Option 2 (from builder) Add the following entry to the `geocoder` entry of the `cartodb/config/app_config.yml` file: ``` @@ -302,3 +302,9 @@ enabled: routing: false data_observatory: true ``` + +Execute the rake tasks to update all the users and organizations: +``` +bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['', true] +bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users['', true] +``` From 00424a0314322b0a5335088e578a9c782f2ee3ca Mon Sep 17 00:00:00 2001 From: Antonio Date: Thu, 24 Aug 2017 10:05:23 +0200 Subject: [PATCH 3/4] Added suggested changes --- README.md | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/README.md b/README.md index 1db1d33..aa962d4 100644 --- a/README.md +++ b/README.md @@ -284,27 +284,4 @@ ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservi #### Option 2 (from builder) -Add the following entry to the `geocoder` entry of the `cartodb/config/app_config.yml` file: -``` -api: - host: 'localhost' - port: '5432' - user: 'dataservices_user' - dbname: 'dataservices_db' -``` - -In the `cartodb/config/app_config.yml` file, enable the desired dataservices: -``` -enabled: - geocoder_internal: false - hires_geocoder: false - isolines: false - routing: false - data_observatory: true -``` - -Execute the rake tasks to update all the users and organizations: -``` -bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['', true] -bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users['', true] -``` +See [the **Configuring Dataservices** documentation](http://cartodb.readthedocs.io/en/latest/operations/configure_data_services.html) \ No newline at end of file From c13dba71a46c5bedf090529477655d67929c759c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 24 Aug 2017 14:35:20 +0200 Subject: [PATCH 4/4] Deactivated DO credits usage for users that have quota See https://github.com/CartoDB/bigmetadata/issues/215 --- .../cartodb_services/metrics/quota.py | 5 ++++- server/lib/python/cartodb_services/setup.py | 2 +- .../cartodb_services/test/test_quota_service.py | 11 +++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index b372ae9..3e6019b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -134,7 +134,10 @@ class QuotaChecker: service_type = self._user_service_config.service_type current_used = self._user_service.used_quota(service_type, today) - if soft_limit or (user_quota > 0 and current_used < user_quota): + # Quick workaround so we don't take into account numer of credits + # spent for users that have defined the quota. + # See https://github.com/CartoDB/bigmetadata/issues/215 + if soft_limit or (user_quota > 0): return True else: return False diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index e0be322..5c1e052 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.15.2', + version='0.15.3', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 7c29559..9bd3e4f 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -126,12 +126,15 @@ class TestQuotaService(TestCase): qs.increment_isolines_service_use(amount=1500000) assert qs.check_user_quota() is False + # Quick workaround so we don't take into account numer of credits + # spent for users that have defined the quota. + # See https://github.com/CartoDB/bigmetadata/issues/215 def test_should_check_user_obs_snapshot_quota_correctly(self): qs = self.__build_obs_snapshot_quota_service('test_user') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is False + assert qs.check_user_quota() is True def test_should_check_org_obs_snapshot_quota_correctly(self): qs = self.__build_obs_snapshot_quota_service('test_user', @@ -139,14 +142,14 @@ class TestQuotaService(TestCase): qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is False + assert qs.check_user_quota() is True def test_should_check_user_obs_quota_correctly(self): qs = self.__build_obs_snapshot_quota_service('test_user') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is False + assert qs.check_user_quota() is True def test_should_check_org_obs_quota_correctly(self): qs = self.__build_obs_quota_service('test_user', @@ -154,7 +157,7 @@ class TestQuotaService(TestCase): qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) - assert qs.check_user_quota() is False + assert qs.check_user_quota() is True def __prepare_quota_service(self, username, service, quota, provider, orgname, soft_limit, end_date):