From 4169bc0f88ef922aed45f37d00de0ac57b1f686a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 8 Nov 2016 17:52:18 +0100 Subject: [PATCH 01/32] Fix doc: how to run python unit tests #302 --- server/lib/python/cartodb_services/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/README.md b/server/lib/python/cartodb_services/README.md index 0823092..197ccd4 100644 --- a/server/lib/python/cartodb_services/README.md +++ b/server/lib/python/cartodb_services/README.md @@ -29,12 +29,12 @@ NOTE: a system installation is required at present because the library is meant ## Running the unit tests -Just run `nosetests` +Just run `nosetests test/` ```shell -$ nosetests -................................................. +$ nosetests test/ +...................................................................................................... ---------------------------------------------------------------------- -Ran 49 tests in 0.131s +Ran 102 tests in 0.122s OK ``` From 2c76fa4501e8939ba85f3404828767a221c7d1d7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 8 Nov 2016 18:28:28 +0100 Subject: [PATCH 02/32] Fix: quotas should be integers #302 --- .../cartodb_services/cartodb_services/metrics/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 897b179..7cdac6a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -94,7 +94,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig): self._soft_limit = False self._monthly_quota = 0 if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -118,7 +118,7 @@ class ObservatoryConfig(DataObservatoryConfig): self._soft_limit = False self._monthly_quota = 0 if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -218,7 +218,7 @@ class IsolinesRoutingConfig(ServiceConfig): self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if self._isolines_provider == self.HEREMAPS_PROVIDER: - self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) + self._isolines_quota = int(filtered_config[self.QUOTA_KEY]) self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': @@ -361,7 +361,7 @@ class GeocoderConfig(ServiceConfig): self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower() else: self._geocoder_provider = self.DEFAULT_PROVIDER - self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) + self._geocoding_quota = int(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True From 6f9feb07a038687bfa27edc8bd1762df93e50b12 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 8 Nov 2016 18:29:06 +0100 Subject: [PATCH 03/32] Add cdb_monthly_quota (WIP) #302 --- server/extension/sql/200_quotas.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 server/extension/sql/200_quotas.sql diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql new file mode 100644 index 0000000..a2e627f --- /dev/null +++ b/server/extension/sql/200_quotas.sql @@ -0,0 +1,15 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_monthly_quota( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS integer AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + if service == 'isolines': + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + return user_isolines_config.isolines_quota + else: + raise 'not implemented' +$$ LANGUAGE plpythonu; From b00edf218ab0614328f19c1971f1ef739eb991ce Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 8 Nov 2016 18:46:58 +0100 Subject: [PATCH 04/32] Fix: quotas should be integers #302 --- .../python/cartodb_services/cartodb_services/metrics/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 1c31aae..bc78f18 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -88,11 +88,11 @@ class UserMetricsService: redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, service, metric, date) score = self._redis_connection.zscore(redis_prefix, date.day) - aggregated_metric += score if score else 0 + aggregated_metric += int(score) if score else 0 zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED) if str(date.day) != zero_padded_day: score = self._redis_connection.zscore(redis_prefix, zero_padded_day) - aggregated_metric += score if score else 0 + aggregated_metric += int(score) if score else 0 return aggregated_metric From 0e38af6ecd3277cdf00f1d13631a2a32d220d80a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 8 Nov 2016 18:47:22 +0100 Subject: [PATCH 05/32] Add cdb_remaining_quota to server #302 --- server/extension/sql/200_quotas.sql | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index a2e627f..7c65869 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -13,3 +13,27 @@ RETURNS integer AS $$ else: raise 'not implemented' $$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_remaining_quota( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS integer AS $$ + from datetime import date + from cartodb_services.metrics.user import UserMetricsService + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + if service == 'isolines': + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + monthly_quota = user_isolines_config.isolines_quota + + user_service = UserMetricsService(user_isolines_config, redis_conn) + current_used = user_service.used_quota(user_isolines_config.service_type, date.today()) + return monthly_quota - current_used + else: + raise 'not implemented' +$$ LANGUAGE plpythonu; From 3e059003b6402c47ab4c6df8e30f8bbf93d40dd2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 11:09:40 +0100 Subject: [PATCH 06/32] Add soft limit function #302 --- server/extension/sql/200_quotas.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 7c65869..0f5fec2 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -37,3 +37,20 @@ RETURNS integer AS $$ else: raise 'not implemented' $$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_soft_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS boolean AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + if service == 'isolines': + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + return user_isolines_config.soft_isolines_limit + else: + raise 'not implemented' +$$ LANGUAGE plpythonu; From eb7188235d95d034d43ec2691c3ba441d01c47d9 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 11:14:15 +0100 Subject: [PATCH 07/32] Provider function #302 --- server/extension/sql/200_quotas.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 0f5fec2..c5d4f36 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -54,3 +54,20 @@ RETURNS boolean AS $$ else: raise 'not implemented' $$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_provider( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS TEXT AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + if service == 'isolines': + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + return user_isolines_config.provider + else: + raise 'not implemented' +$$ LANGUAGE plpythonu; From 7a56dc698f41f3ff387b1372295efe3d2962f480 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 16:59:58 +0100 Subject: [PATCH 08/32] List of services bound to quota #302 --- .../cartodb_services/cartodb_services/metrics/quota.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index b372ae9..3893fab 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -3,6 +3,15 @@ from log import MetricsDataGatherer from datetime import date import re +class Service: + """The services bound to quota""" + + ISOLINES = 'isolines' + HIRES_GEOCODER = 'hires_geocoder' + ROUTING = 'routing' + OBSERVATORY = 'observatory' + INTERNAL_GEOCODER = 'internal_geocoder' + class QuotaService: """ Class to manage all the quota operation for From 7a32ace1aca220fbe42868a524142ee3ef4f73e8 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 17:00:25 +0100 Subject: [PATCH 09/32] Add the hires_geocoder to cdb_monthly_quota #302 --- server/extension/sql/200_quotas.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index c5d4f36..93d77d9 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -3,15 +3,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_monthly_quota( orgname TEXT, service TEXT) RETURNS integer AS $$ + import cartodb_services.metrics.quota as quota plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - if service == 'isolines': + if service == quota.Service.ISOLINES: plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - return user_isolines_config.isolines_quota + monthly_quota = user_isolines_config.isolines_quota + elif service == quota.Service.HIRES_GEOCODER: + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + monthly_quota = user_geocoder_config.geocoding_quota else: raise 'not implemented' + + return monthly_quota $$ LANGUAGE plpythonu; From 84061dec4f56842ccf2a780b367978a0a48474bf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 18:12:24 +0100 Subject: [PATCH 10/32] Move everything to cdb_service_params (WIP) #302 --- server/extension/sql/200_quotas.sql | 76 ++++++++++++++--------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 93d77d9..8ba354b 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -1,3 +1,39 @@ +CREATE TYPE cdb_dataservices_server.service_params AS ( + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_params( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS cdb_dataservices_server.service_params AS $$ + import cartodb_services.metrics.quota as quota + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + if service == quota.Service.ISOLINES: + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + monthly_quota = user_isolines_config.isolines_quota + user_service = UserMetricsService(user_isolines_config, redis_conn) + used_quota = user_service.used_quota(user_isolines_config.service_type, date.today()) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + else: + raise 'not implemented' + + return [monthly_quota, used_quota, soft_limit, provider] + +$$ LANGUAGE plpythonu; + + CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_monthly_quota( username TEXT, orgname TEXT, @@ -7,11 +43,7 @@ RETURNS integer AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - if service == quota.Service.ISOLINES: - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - monthly_quota = user_isolines_config.isolines_quota - elif service == quota.Service.HIRES_GEOCODER: + if service == quota.Service.HIRES_GEOCODER: plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] monthly_quota = user_geocoder_config.geocoding_quota @@ -44,37 +76,3 @@ RETURNS integer AS $$ else: raise 'not implemented' $$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_soft_limit( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS boolean AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - if service == 'isolines': - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - return user_isolines_config.soft_isolines_limit - else: - raise 'not implemented' -$$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_provider( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS TEXT AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - if service == 'isolines': - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - return user_isolines_config.provider - else: - raise 'not implemented' -$$ LANGUAGE plpythonu; From b2cdb1c74b0ae4ff0143dd7243cb7af896fea831 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 18:40:18 +0100 Subject: [PATCH 11/32] Move all into one func returning a tuple #302 --- server/extension/sql/200_quotas.sql | 60 +++++++---------------------- 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 8ba354b..81676e8 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -18,61 +18,29 @@ RETURNS cdb_dataservices_server.service_params AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + today = date.today() + if service == quota.Service.ISOLINES: plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - monthly_quota = user_isolines_config.isolines_quota user_service = UserMetricsService(user_isolines_config, redis_conn) - used_quota = user_service.used_quota(user_isolines_config.service_type, date.today()) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) soft_limit = user_isolines_config.soft_isolines_limit provider = user_isolines_config.provider + elif service == quota.Service.HIRES_GEOCODER: + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider else: raise 'not implemented' return [monthly_quota, used_quota, soft_limit, provider] $$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_monthly_quota( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS integer AS $$ - import cartodb_services.metrics.quota as quota - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - if service == quota.Service.HIRES_GEOCODER: - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - monthly_quota = user_geocoder_config.geocoding_quota - else: - raise 'not implemented' - - return monthly_quota -$$ LANGUAGE plpythonu; - - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_remaining_quota( - username TEXT, - orgname TEXT, - service TEXT) -RETURNS integer AS $$ - from datetime import date - from cartodb_services.metrics.user import UserMetricsService - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - - if service == 'isolines': - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - monthly_quota = user_isolines_config.isolines_quota - - user_service = UserMetricsService(user_isolines_config, redis_conn) - current_used = user_service.used_quota(user_isolines_config.service_type, date.today()) - return monthly_quota - current_used - else: - raise 'not implemented' -$$ LANGUAGE plpythonu; From 429a71ef370f5f848fd96c3fed6e991584414841 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 18:56:50 +0100 Subject: [PATCH 12/32] Add a function to check for enough quota #302 --- server/extension/sql/200_quotas.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 81676e8..22d270b 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -44,3 +44,18 @@ RETURNS cdb_dataservices_server.service_params AS $$ return [monthly_quota, used_quota, soft_limit, provider] $$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_params; + BEGIN + SELECT * INTO params FROM cdb_dataservices_server.cdb_service_params(username, orgname, service); + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql; From 3fa686ec652ced023fc6fe099232c263da66799c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 19:21:29 +0100 Subject: [PATCH 13/32] Remove the class Service in favor of service_type ENUM #302 --- server/extension/sql/200_quotas.sql | 8 ++++++++ .../cartodb_services/cartodb_services/metrics/quota.py | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 22d270b..9245337 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -1,3 +1,11 @@ +CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory', + 'internal_geocoder' +); + CREATE TYPE cdb_dataservices_server.service_params AS ( monthly_quota NUMERIC, used_quota NUMERIC, diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 3893fab..b372ae9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -3,15 +3,6 @@ from log import MetricsDataGatherer from datetime import date import re -class Service: - """The services bound to quota""" - - ISOLINES = 'isolines' - HIRES_GEOCODER = 'hires_geocoder' - ROUTING = 'routing' - OBSERVATORY = 'observatory' - INTERNAL_GEOCODER = 'internal_geocoder' - class QuotaService: """ Class to manage all the quota operation for From 603fbbbc3f11375bfd0e319c90f93da166d8a604 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 19:23:01 +0100 Subject: [PATCH 14/32] Make cdb_service_params return a table for all services #302 --- server/extension/sql/200_quotas.sql | 48 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 9245337..0b57781 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -7,6 +7,7 @@ CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( ); CREATE TYPE cdb_dataservices_server.service_params AS ( + service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, soft_limit BOOLEAN, @@ -16,10 +17,8 @@ CREATE TYPE cdb_dataservices_server.service_params AS ( CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_params( username TEXT, - orgname TEXT, - service TEXT) -RETURNS cdb_dataservices_server.service_params AS $$ - import cartodb_services.metrics.quota as quota + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_params AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date @@ -27,30 +26,33 @@ RETURNS cdb_dataservices_server.service_params AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] today = date.today() + ret = [] - if service == quota.Service.ISOLINES: - plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] - user_service = UserMetricsService(user_isolines_config, redis_conn) + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) - monthly_quota = user_isolines_config.isolines_quota - used_quota = user_service.used_quota(user_isolines_config.service_type, today) - soft_limit = user_isolines_config.soft_isolines_limit - provider = user_isolines_config.provider - elif service == quota.Service.HIRES_GEOCODER: - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config, today) - soft_limit = user_geocoder_config.soft_geocoding_limit - provider = user_geocoder_config.provider - else: - raise 'not implemented' + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) - return [monthly_quota, used_quota, soft_limit, provider] + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + return ret $$ LANGUAGE plpythonu; From 9f9034f4f177515194fe393d4c3b9886ab9fb7c3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 10 Nov 2016 19:23:39 +0100 Subject: [PATCH 15/32] Make cdb_enough_quota check the table got from cdb_service_params #302 --- server/extension/sql/200_quotas.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 0b57781..58f49c6 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -59,13 +59,15 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, - service TEXT, + service_ cdb_dataservices_server.service_type, input_size NUMERIC) returns BOOLEAN AS $$ DECLARE params cdb_dataservices_server.service_params; BEGIN - SELECT * INTO params FROM cdb_dataservices_server.cdb_service_params(username, orgname, service); + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_params(username, orgname) AS p + WHERE p.service = service_; RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); END $$ LANGUAGE plpgsql; From 95dda082e7404dfd4447f685c922ff907eddc56d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 11 Nov 2016 10:24:03 +0100 Subject: [PATCH 16/32] Add routing #302 --- server/extension/sql/200_quotas.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 58f49c6..177d8c6 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -52,6 +52,18 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ provider = user_geocoder_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + return ret $$ LANGUAGE plpythonu; From 2e6f37af5607243ef993a0f51fc998b8fa19c746 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 11 Nov 2016 11:27:57 +0100 Subject: [PATCH 17/32] Add observatory #302 --- server/extension/sql/200_quotas.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 177d8c6..f8fe8cf 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -64,6 +64,18 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ provider = user_routing_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + return ret $$ LANGUAGE plpythonu; From abbb227ad5f62a6707b5cd38de8ee14a00e3d329 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 11:40:00 +0100 Subject: [PATCH 18/32] Fix typo: use obs config for obs service #302 --- server/extension/sql/200_quotas.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index f8fe8cf..cc2a09e 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -71,7 +71,7 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ user_service = UserMetricsService(user_obs_config, redis_conn) monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config, today) + used_quota = user_service.used_quota(user_obs_config, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] From decc7626a8c58de4afb59f94b9f5fdcba53ccdd3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 16:38:47 +0100 Subject: [PATCH 19/32] Create types only if they don't exist #302 --- server/extension/sql/200_quotas.sql | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index cc2a09e..66f6914 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -1,19 +1,28 @@ -CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( - 'isolines', - 'hires_geocoder', - 'routing', - 'observatory', - 'internal_geocoder' -); - -CREATE TYPE cdb_dataservices_server.service_params AS ( - service cdb_dataservices_server.service_type, - monthly_quota NUMERIC, - used_quota NUMERIC, - soft_limit BOOLEAN, - provider TEXT -); +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory', + 'internal_geocoder' + ); + END IF; +END $$; +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_params') THEN + CREATE TYPE cdb_dataservices_server.service_params AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_params( username TEXT, From d70d149eed0cbc128f6d3329f9caafc93848fdc5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 16:40:00 +0100 Subject: [PATCH 20/32] Fix bug: the service_type to used_quota #302 --- server/extension/sql/200_quotas.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 66f6914..0b8ccd2 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -56,7 +56,7 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ user_service = UserMetricsService(user_geocoder_config, redis_conn) monthly_quota = user_geocoder_config.geocoding_quota - used_quota = user_service.used_quota(user_geocoder_config, today) + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) soft_limit = user_geocoder_config.soft_geocoding_limit provider = user_geocoder_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] @@ -68,7 +68,7 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ user_service = UserMetricsService(user_routing_config, redis_conn) monthly_quota = user_routing_config.monthly_quota - used_quota = user_service.used_quota(user_routing_config, today) + used_quota = user_service.used_quota(user_routing_config.service_type, today) soft_limit = user_routing_config.soft_limit provider = user_routing_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] @@ -80,7 +80,7 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ user_service = UserMetricsService(user_obs_config, redis_conn) monthly_quota = user_obs_config.monthly_quota - used_quota = user_service.used_quota(user_obs_config, today) + used_quota = user_service.used_quota(user_obs_config.service_type, today) soft_limit = user_obs_config.soft_limit provider = user_obs_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] From a1eae3934c5404925cc141dd29fd654ee93b825a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 16:47:35 +0100 Subject: [PATCH 21/32] Add OBS to UserMetricsService #302 --- .../cartodb_services/metrics/user.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index bc78f18..4a65014 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -9,6 +9,7 @@ class UserMetricsService: SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' + SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' def __init__(self, user_geocoder_config, redis_connection): @@ -22,6 +23,8 @@ class UserMetricsService: return self.__used_isolines_quota(service_type, date) elif service_type == self.SERVICE_MAPZEN_ROUTING: return self.__used_routing_quota(service_type, date) + elif service_type == self.SERVICE_OBSERVATORY: + return self.__used_observatory_quota(service_type, date) else: return self.__used_geocoding_quota(service_type, date) @@ -72,6 +75,19 @@ class UserMetricsService: return current_use + def __used_observatory_quota(self, service_type, date): + date_from, date_to = self.__current_billing_cycle() + current_use = 0 + success_responses = self.get_metrics(service_type, + 'success_responses', date_from, + date_to) + empty_responses = self.get_metrics(service_type, + 'empty_responses', date_from, + date_to) + current_use += (success_responses + empty_responses) + + return current_use + def increment_service_use(self, service_type, metric, date=date.today(), amount=1): """ Increment the services uses in monthly and daily basis""" From 579d11ebd5775680ea55f3c1e87d192a819853cd Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 17:04:48 +0100 Subject: [PATCH 22/32] Add params to InternalGeocoderConfig #302 Add a bunch of params to InternalGeocoderConfig for consistency with other services: - monthly_quota - period_end_date - soft_limit --- .../cartodb_services/metrics/config.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 7cdac6a..1bc7011 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -281,11 +281,13 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): METRICS_LOG_KEY = 'geocoder_log_path' + PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, db_conn, username, orgname=None): # For now, internal geocoder doesn't use the redis config - super(InternalGeocoderConfig, self).__init__(None, db_conn, + super(InternalGeocoderConfig, self).__init__(redis_connection, db_conn, username, orgname) + self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property def service_type(self): @@ -303,10 +305,23 @@ class InternalGeocoderConfig(ServiceConfig): def geocoding_quota(self): return None + @property + def monthly_quota(self): + return None + @property def provider(self): return 'internal' + @property + def period_end_date(self): + return self._period_end_date + + @property + def soft_limit(self): + return False + + class GeocoderConfig(ServiceConfig): From 4af5975ac14d49ee768f50c9fca4efb0eb1be0d7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 17:07:23 +0100 Subject: [PATCH 23/32] Add the internal geocoder to cdb_service_params #302 --- server/extension/sql/200_quotas.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 0b8ccd2..925dc3d 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -85,6 +85,18 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ provider = user_obs_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + #-- Internal geocoder + service = 'internal_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.monthly_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + return ret $$ LANGUAGE plpythonu; From a2da597e0021ad285dc24665a91909f7faf8c07a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 18:10:44 +0100 Subject: [PATCH 24/32] Support null quota (internal_geocoder) #302 --- server/extension/sql/200_quotas.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 925dc3d..65541cd 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -113,6 +113,8 @@ returns BOOLEAN AS $$ SELECT * INTO params FROM cdb_dataservices_server.cdb_service_params(username, orgname) AS p WHERE p.service = service_; - RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + RETURN params.soft_limit + OR params.monthly_quota IS NULL -- account for the internal_geocoder + OR ((params.used_quota + input_size) <= params.monthly_quota); END $$ LANGUAGE plpgsql; From 1dc6060570140ea99aea413fc0f0583349bb7858 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 23 Nov 2016 18:51:06 +0100 Subject: [PATCH 25/32] Fix: deal with string values for soft limits #302 --- .../cartodb_services/refactor/service/mapzen_geocoder_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 71e2d35..5ce8f56 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -92,7 +92,7 @@ class MapzenGeocoderConfigBuilder(object): geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] - soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit') + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' cost_per_hit=0 From 37a3214f67ebba75d16def7cfc24e8428f35491a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 13:05:31 +0100 Subject: [PATCH 26/32] Add quota functions to client #302 --- client/renderer/interface.yaml | 13 +++++++++++++ client/sql/16_custom_types.sql | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 4abe429..be758f9 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -377,3 +377,16 @@ multi_field: true params: - { name: aggregate_type, type: text, default: 'NULL' } + +- name: cdb_service_params + return_type: SETOF cdb_dataservices_client.service_params + multi_row: true + no_params: true + params: + - {} + +- name: cdb_enough_quota + return_type: BOOLEAN + params: + - { name: service, type: cdb_dataservices_client.service_type } + - { name: input_size, type: NUMERIC } \ No newline at end of file diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 0bb19a2..c400443 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -18,3 +18,21 @@ CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, deno CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric); CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory', + 'internal_geocoder' +); + +CREATE TYPE cdb_dataservices_client.service_params AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); From f07d2f93029982bac38b0b8d7d591a1c8e8e556a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 13:27:32 +0100 Subject: [PATCH 27/32] Remove the no_params from client generator #302 --- client/renderer/interface.yaml | 2 -- client/renderer/sql-template-renderer | 10 +++------- client/renderer/templates/20_public_functions.erb | 11 ++++------- client/renderer/templates/30_plproxy_functions.erb | 12 +++--------- client/renderer/templates/90_grant_execute.erb | 2 +- 5 files changed, 11 insertions(+), 26 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index be758f9..a40c99d 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -323,7 +323,6 @@ - name: obs_dumpversion return_type: text - no_params: true params: - {} @@ -381,7 +380,6 @@ - name: cdb_service_params return_type: SETOF cdb_dataservices_client.service_params multi_row: true - no_params: true params: - {} diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index af558d0..2516a09 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -36,10 +36,6 @@ class SqlTemplateRenderer @function_signature['multi_row'] end - def no_params - @function_signature['no_params'] - end - def user_config_key @function_signature['user_config_key'] end @@ -49,11 +45,11 @@ class SqlTemplateRenderer end def params - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}.join(', ') + @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"} end def params_with_type - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }.join(', ') + @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" } end def params_with_type_and_default @@ -64,7 +60,7 @@ class SqlTemplateRenderer "#{p['name']} #{p['type']}" end end - return parameters.join(', ') + return parameters end end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 5de10bd..88db53d 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -4,7 +4,7 @@ -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>) +CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>) RETURNS <%= return_type %> AS $$ DECLARE <% if not multi_row %>ret <%= return_type %>;<% end %> @@ -21,15 +21,12 @@ BEGIN END IF; <% if multi_row %> RETURN QUERY - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>); + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>); <% elsif multi_field %> - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; - RETURN ret; - <% elsif no_params %> - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname) INTO ret; + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; RETURN ret; <% else %> - SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; + SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; RETURN ret; <% end %> END; diff --git a/client/renderer/templates/30_plproxy_functions.erb b/client/renderer/templates/30_plproxy_functions.erb index 66a6727..2d5dfe3 100644 --- a/client/renderer/templates/30_plproxy_functions.erb +++ b/client/renderer/templates/30_plproxy_functions.erb @@ -1,15 +1,9 @@ -<% if no_params %> -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text) -<% else %> -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) -<% end %> +CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>) RETURNS <%= return_type %> AS $$ CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str(); <% if multi_field %> - SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); - <% elsif no_params %> - SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name); + SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); <% else %> - SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); + SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); <% end %> $$ LANGUAGE plproxy; diff --git a/client/renderer/templates/90_grant_execute.erb b/client/renderer/templates/90_grant_execute.erb index 7d0c030..bde5944 100644 --- a/client/renderer/templates/90_grant_execute.erb +++ b/client/renderer/templates/90_grant_execute.erb @@ -1 +1 @@ -GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type %>) TO publicuser; \ No newline at end of file +GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser; From 4c75c5f2610a8f0b22da99b8f0294605a6dd614f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 17:33:06 +0100 Subject: [PATCH 28/32] Revert internal geocoder related stuff #302 This reverts commits 579d11e, 4af5975 and a2da597 --- server/extension/sql/200_quotas.sql | 16 +--------------- .../cartodb_services/metrics/config.py | 17 +---------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 65541cd..0b8ccd2 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -85,18 +85,6 @@ RETURNS SETOF cdb_dataservices_server.service_params AS $$ provider = user_obs_config.provider ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - #-- Internal geocoder - service = 'internal_geocoder' - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - user_service = UserMetricsService(user_geocoder_config, redis_conn) - - monthly_quota = user_geocoder_config.monthly_quota - used_quota = user_service.used_quota(user_geocoder_config.service_type, today) - soft_limit = user_geocoder_config.soft_limit - provider = user_geocoder_config.provider - ret += [[service, monthly_quota, used_quota, soft_limit, provider]] - return ret $$ LANGUAGE plpythonu; @@ -113,8 +101,6 @@ returns BOOLEAN AS $$ SELECT * INTO params FROM cdb_dataservices_server.cdb_service_params(username, orgname) AS p WHERE p.service = service_; - RETURN params.soft_limit - OR params.monthly_quota IS NULL -- account for the internal_geocoder - OR ((params.used_quota + input_size) <= params.monthly_quota); + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); END $$ LANGUAGE plpgsql; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 1bc7011..7cdac6a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -281,13 +281,11 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): METRICS_LOG_KEY = 'geocoder_log_path' - PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, db_conn, username, orgname=None): # For now, internal geocoder doesn't use the redis config - super(InternalGeocoderConfig, self).__init__(redis_connection, db_conn, + super(InternalGeocoderConfig, self).__init__(None, db_conn, username, orgname) - self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @property def service_type(self): @@ -305,23 +303,10 @@ class InternalGeocoderConfig(ServiceConfig): def geocoding_quota(self): return None - @property - def monthly_quota(self): - return None - @property def provider(self): return 'internal' - @property - def period_end_date(self): - return self._period_end_date - - @property - def soft_limit(self): - return False - - class GeocoderConfig(ServiceConfig): From c2ede37d75e416f197862f4001de5d4622d39b77 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 17:43:58 +0100 Subject: [PATCH 29/32] Revert internal geocoder related stuff #302 --- client/sql/16_custom_types.sql | 3 +-- server/extension/sql/200_quotas.sql | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index c400443..a316d8e 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -25,8 +25,7 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( 'isolines', 'hires_geocoder', 'routing', - 'observatory', - 'internal_geocoder' + 'observatory' ); CREATE TYPE cdb_dataservices_client.service_params AS ( diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 0b8ccd2..34f5279 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -5,8 +5,7 @@ BEGIN 'isolines', 'hires_geocoder', 'routing', - 'observatory', - 'internal_geocoder' + 'observatory' ); END IF; END $$; From 769b740ba40357d3eabaf65bf7b5fb7dd7a636e7 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 17:46:07 +0100 Subject: [PATCH 30/32] Rename service_params to service_quota_info #302 --- client/renderer/interface.yaml | 4 ++-- client/sql/16_custom_types.sql | 2 +- server/extension/sql/200_quotas.sql | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index a40c99d..8c33801 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -377,8 +377,8 @@ params: - { name: aggregate_type, type: text, default: 'NULL' } -- name: cdb_service_params - return_type: SETOF cdb_dataservices_client.service_params +- name: cdb_service_quota_info + return_type: SETOF cdb_dataservices_client.service_quota_info multi_row: true params: - {} diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index a316d8e..37e62f9 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -28,7 +28,7 @@ CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( 'observatory' ); -CREATE TYPE cdb_dataservices_client.service_params AS ( +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( service cdb_dataservices_client.service_type, monthly_quota NUMERIC, used_quota NUMERIC, diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index 34f5279..c0aefca 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -12,8 +12,8 @@ END $$; DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_params') THEN - CREATE TYPE cdb_dataservices_server.service_params AS ( + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_quota_info') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( service cdb_dataservices_server.service_type, monthly_quota NUMERIC, used_quota NUMERIC, @@ -23,10 +23,10 @@ BEGIN END IF; END $$; -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_params( +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( username TEXT, orgname TEXT) -RETURNS SETOF cdb_dataservices_server.service_params AS $$ +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ from cartodb_services.metrics.user import UserMetricsService from datetime import date @@ -95,10 +95,10 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( input_size NUMERIC) returns BOOLEAN AS $$ DECLARE - params cdb_dataservices_server.service_params; + params cdb_dataservices_server.service_quota_info; BEGIN SELECT * INTO params - FROM cdb_dataservices_server.cdb_service_params(username, orgname) AS p + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p WHERE p.service = service_; RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); END From 0d92eb4ba8ae9bddcce8bd86f55fd8f72e769919 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 18:24:13 +0100 Subject: [PATCH 31/32] Fix issue with multi-field values #302 The difference between ``` => select * from cdb_service_quota_info(); service | monthly_quota | used_quota | soft_limit | provider ----------------+---------------+------------+------------+------------------ isolines | 1000 | 0 | f | heremaps hires_geocoder | 5 | 2 | f | mapzen routing | 1500000 | 0 | f | mapzen observatory | 1000 | 0 | f | data observatory (4 rows) ``` and ``` => select cdb_service_quota_info(); cdb_service_quota_info ------------------------------------------- (isolines,1000,0,f,heremaps) (hires_geocoder,5,2,f,mapzen) (routing,1500000,0,f,mapzen) (observatory,1000,0,f,"data observatory") (4 rows) ``` is important to pl/proxy. In the later case, rows only have one field (a tuple) and it complains with "Got too few fields from remote end". --- client/renderer/interface.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 8c33801..d70d4ad 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -380,6 +380,7 @@ - name: cdb_service_quota_info return_type: SETOF cdb_dataservices_client.service_quota_info multi_row: true + multi_field: true params: - {} From 9c6eabc59efb2ff08f820d7059ff5afcb2e5cde3 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 24 Nov 2016 18:38:31 +0100 Subject: [PATCH 32/32] Do not enforce types for params #302 As types are tied to a schema, they don't get along very well with pl/proxy. Do not use them for the service_type. --- client/renderer/interface.yaml | 4 ++-- server/extension/sql/200_quotas.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index d70d4ad..3b9ff7c 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -378,7 +378,7 @@ - { name: aggregate_type, type: text, default: 'NULL' } - name: cdb_service_quota_info - return_type: SETOF cdb_dataservices_client.service_quota_info + return_type: SETOF service_quota_info multi_row: true multi_field: true params: @@ -387,5 +387,5 @@ - name: cdb_enough_quota return_type: BOOLEAN params: - - { name: service, type: cdb_dataservices_client.service_type } + - { name: service, type: TEXT } - { name: input_size, type: NUMERIC } \ No newline at end of file diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql index c0aefca..48b84c2 100644 --- a/server/extension/sql/200_quotas.sql +++ b/server/extension/sql/200_quotas.sql @@ -91,7 +91,7 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( username TEXT, orgname TEXT, - service_ cdb_dataservices_server.service_type, + service_ TEXT, input_size NUMERIC) returns BOOLEAN AS $$ DECLARE @@ -99,7 +99,7 @@ returns BOOLEAN AS $$ BEGIN SELECT * INTO params FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p - WHERE p.service = service_; + WHERE p.service = service_::cdb_dataservices_server.service_type; RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); END $$ LANGUAGE plpgsql;