diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index 4abe429..3b9ff7c 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -323,7 +323,6 @@ - name: obs_dumpversion return_type: text - no_params: true params: - {} @@ -377,3 +376,16 @@ multi_field: true params: - { name: aggregate_type, type: text, default: 'NULL' } + +- name: cdb_service_quota_info + return_type: SETOF service_quota_info + multi_row: true + multi_field: true + params: + - {} + +- name: cdb_enough_quota + return_type: BOOLEAN + params: + - { name: service, type: TEXT } + - { name: input_size, type: NUMERIC } \ No newline at end of file diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index af558d0..2516a09 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -36,10 +36,6 @@ class SqlTemplateRenderer @function_signature['multi_row'] end - def no_params - @function_signature['no_params'] - end - def user_config_key @function_signature['user_config_key'] end @@ -49,11 +45,11 @@ class SqlTemplateRenderer end def params - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}.join(', ') + @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"} end def params_with_type - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }.join(', ') + @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" } end def params_with_type_and_default @@ -64,7 +60,7 @@ class SqlTemplateRenderer "#{p['name']} #{p['type']}" end end - return parameters.join(', ') + return parameters end end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 5de10bd..88db53d 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -4,7 +4,7 @@ -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>) +CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>) RETURNS <%= return_type %> AS $$ DECLARE <% if not multi_row %>ret <%= return_type %>;<% end %> @@ -21,15 +21,12 @@ BEGIN END IF; <% if multi_row %> RETURN QUERY - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>); + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>); <% elsif multi_field %> - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; - RETURN ret; - <% elsif no_params %> - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname) INTO ret; + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; RETURN ret; <% else %> - SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; + SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; RETURN ret; <% end %> END; diff --git a/client/renderer/templates/30_plproxy_functions.erb b/client/renderer/templates/30_plproxy_functions.erb index 66a6727..2d5dfe3 100644 --- a/client/renderer/templates/30_plproxy_functions.erb +++ b/client/renderer/templates/30_plproxy_functions.erb @@ -1,15 +1,9 @@ -<% if no_params %> -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text) -<% else %> -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) -<% end %> +CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>) RETURNS <%= return_type %> AS $$ CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str(); <% if multi_field %> - SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); - <% elsif no_params %> - SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name); + SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); <% else %> - SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); + SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); <% end %> $$ LANGUAGE plproxy; diff --git a/client/renderer/templates/90_grant_execute.erb b/client/renderer/templates/90_grant_execute.erb index 7d0c030..bde5944 100644 --- a/client/renderer/templates/90_grant_execute.erb +++ b/client/renderer/templates/90_grant_execute.erb @@ -1 +1 @@ -GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type %>) TO publicuser; \ No newline at end of file +GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser; diff --git a/client/sql/16_custom_types.sql b/client/sql/16_custom_types.sql index 0bb19a2..37e62f9 100644 --- a/client/sql/16_custom_types.sql +++ b/client/sql/16_custom_types.sql @@ -18,3 +18,20 @@ CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, deno CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric); CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean); + + +-- For quotas and services configuration +CREATE TYPE cdb_dataservices_client.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' +); + +CREATE TYPE cdb_dataservices_client.service_quota_info AS ( + service cdb_dataservices_client.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT +); diff --git a/server/extension/sql/200_quotas.sql b/server/extension/sql/200_quotas.sql new file mode 100644 index 0000000..48b84c2 --- /dev/null +++ b/server/extension/sql/200_quotas.sql @@ -0,0 +1,105 @@ +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN + CREATE TYPE cdb_dataservices_server.service_type AS ENUM ( + 'isolines', + 'hires_geocoder', + 'routing', + 'observatory' + ); + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_quota_info') THEN + CREATE TYPE cdb_dataservices_server.service_quota_info AS ( + service cdb_dataservices_server.service_type, + monthly_quota NUMERIC, + used_quota NUMERIC, + soft_limit BOOLEAN, + provider TEXT + ); + END IF; +END $$; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info( + username TEXT, + orgname TEXT) +RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$ + from cartodb_services.metrics.user import UserMetricsService + from datetime import date + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + today = date.today() + ret = [] + + #-- Isolines + service = 'isolines' + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_isolines_config, redis_conn) + + monthly_quota = user_isolines_config.isolines_quota + used_quota = user_service.used_quota(user_isolines_config.service_type, today) + soft_limit = user_isolines_config.soft_isolines_limit + provider = user_isolines_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Hires Geocoder + service = 'hires_geocoder' + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + user_service = UserMetricsService(user_geocoder_config, redis_conn) + + monthly_quota = user_geocoder_config.geocoding_quota + used_quota = user_service.used_quota(user_geocoder_config.service_type, today) + soft_limit = user_geocoder_config.soft_geocoding_limit + provider = user_geocoder_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Routing + service = 'routing' + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + user_service = UserMetricsService(user_routing_config, redis_conn) + + monthly_quota = user_routing_config.monthly_quota + used_quota = user_service.used_quota(user_routing_config.service_type, today) + soft_limit = user_routing_config.soft_limit + provider = user_routing_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + #-- Observatory + service = 'observatory' + plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_config = GD["user_obs_config_{0}".format(username)] + user_service = UserMetricsService(user_obs_config, redis_conn) + + monthly_quota = user_obs_config.monthly_quota + used_quota = user_service.used_quota(user_obs_config.service_type, today) + soft_limit = user_obs_config.soft_limit + provider = user_obs_config.provider + ret += [[service, monthly_quota, used_quota, soft_limit, provider]] + + return ret +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota( + username TEXT, + orgname TEXT, + service_ TEXT, + input_size NUMERIC) +returns BOOLEAN AS $$ + DECLARE + params cdb_dataservices_server.service_quota_info; + BEGIN + SELECT * INTO params + FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p + WHERE p.service = service_::cdb_dataservices_server.service_type; + RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota); + END +$$ LANGUAGE plpgsql; diff --git a/server/lib/python/cartodb_services/README.md b/server/lib/python/cartodb_services/README.md index 0823092..197ccd4 100644 --- a/server/lib/python/cartodb_services/README.md +++ b/server/lib/python/cartodb_services/README.md @@ -29,12 +29,12 @@ NOTE: a system installation is required at present because the library is meant ## Running the unit tests -Just run `nosetests` +Just run `nosetests test/` ```shell -$ nosetests -................................................. +$ nosetests test/ +...................................................................................................... ---------------------------------------------------------------------- -Ran 49 tests in 0.131s +Ran 102 tests in 0.122s OK ``` diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 897b179..7cdac6a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -94,7 +94,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig): self._soft_limit = False self._monthly_quota = 0 if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -118,7 +118,7 @@ class ObservatoryConfig(DataObservatoryConfig): self._soft_limit = False self._monthly_quota = 0 if self.QUOTA_KEY in self._redis_config: - self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + self._monthly_quota = int(self._redis_config[self.QUOTA_KEY]) self._connection_str = self._db_config.data_observatory_connection_str @property @@ -218,7 +218,7 @@ class IsolinesRoutingConfig(ServiceConfig): self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower() self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if self._isolines_provider == self.HEREMAPS_PROVIDER: - self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) + self._isolines_quota = int(filtered_config[self.QUOTA_KEY]) self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': @@ -361,7 +361,7 @@ class GeocoderConfig(ServiceConfig): self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower() else: self._geocoder_provider = self.DEFAULT_PROVIDER - self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) + self._geocoding_quota = int(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index 1c31aae..4a65014 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -9,6 +9,7 @@ class UserMetricsService: SERVICE_GEOCODER_CACHE = 'geocoder_cache' SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' + SERVICE_OBSERVATORY = 'obs_general' DAY_OF_MONTH_ZERO_PADDED = '%d' def __init__(self, user_geocoder_config, redis_connection): @@ -22,6 +23,8 @@ class UserMetricsService: return self.__used_isolines_quota(service_type, date) elif service_type == self.SERVICE_MAPZEN_ROUTING: return self.__used_routing_quota(service_type, date) + elif service_type == self.SERVICE_OBSERVATORY: + return self.__used_observatory_quota(service_type, date) else: return self.__used_geocoding_quota(service_type, date) @@ -72,6 +75,19 @@ class UserMetricsService: return current_use + def __used_observatory_quota(self, service_type, date): + date_from, date_to = self.__current_billing_cycle() + current_use = 0 + success_responses = self.get_metrics(service_type, + 'success_responses', date_from, + date_to) + empty_responses = self.get_metrics(service_type, + 'empty_responses', date_from, + date_to) + current_use += (success_responses + empty_responses) + + return current_use + def increment_service_use(self, service_type, metric, date=date.today(), amount=1): """ Increment the services uses in monthly and daily basis""" @@ -88,11 +104,11 @@ class UserMetricsService: redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, service, metric, date) score = self._redis_connection.zscore(redis_prefix, date.day) - aggregated_metric += score if score else 0 + aggregated_metric += int(score) if score else 0 zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED) if str(date.day) != zero_padded_day: score = self._redis_connection.zscore(redis_prefix, zero_padded_day) - aggregated_metric += score if score else 0 + aggregated_metric += int(score) if score else 0 return aggregated_metric diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 9361cdb..b37b483 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -92,7 +92,7 @@ class MapzenGeocoderConfigBuilder(object): geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota'] mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] - soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit') + soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' cost_per_hit=0