Merge pull request #311 from CartoDB/302-expose-quotas

302 expose quotas

client/renderer/interface.yaml

@@ -323,7 +323,6 @@
 
 - name: obs_dumpversion
   return_type: text
-  no_params: true
   params:
     - {}
 
@@ -377,3 +376,16 @@
   multi_field: true
   params:
     - { name: aggregate_type, type: text, default: 'NULL' }
+
+- name: cdb_service_quota_info
+  return_type: SETOF service_quota_info
+  multi_row: true
+  multi_field: true
+  params:
+    - {}
+
+- name: cdb_enough_quota
+  return_type: BOOLEAN
+  params:
+    - { name: service, type: TEXT }
+    - { name: input_size, type: NUMERIC }

client/renderer/sql-template-renderer

@@ -36,10 +36,6 @@ class SqlTemplateRenderer
     @function_signature['multi_row']
   end
 
-  def no_params
-    @function_signature['no_params']
-  end
-
   def user_config_key
     @function_signature['user_config_key']
   end
@@ -49,11 +45,11 @@ class SqlTemplateRenderer
   end
 
   def params
-    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}.join(', ')
+    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}
   end
 
   def params_with_type
-    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }.join(', ')
+    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }
   end
 
   def params_with_type_and_default
@@ -64,7 +60,7 @@ class SqlTemplateRenderer
         "#{p['name']} #{p['type']}"
       end
     end
-    return parameters.join(', ')
+    return parameters
   end
 end
 

client/renderer/templates/20_public_functions.erb

@@ -4,7 +4,7 @@
 -- These are the only ones with permissions to publicuser role
 -- and should also be the only ones with SECURITY DEFINER
 
-CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>)
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>)
 RETURNS <%= return_type %> AS $$
 DECLARE
   <% if not multi_row %>ret <%= return_type %>;<% end %>
@@ -21,15 +21,12 @@ BEGIN
   END IF;
   <% if multi_row %>
     RETURN QUERY
-    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>);
+    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
   <% elsif multi_field %>
-    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret;
-    RETURN ret;
-  <% elsif no_params %>
-    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname) INTO ret;
+    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
     RETURN ret;
   <% else %>
-    SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret;
+    SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
     RETURN ret;
   <% end %>
 END;

client/renderer/templates/30_plproxy_functions.erb

@@ -1,15 +1,9 @@
-<% if no_params %>
-CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text)
-<% else %>
-CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>)
-<% end %>
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>)
 RETURNS <%= return_type %> AS $$
   CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str();
   <% if multi_field %>
-  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>);
-  <% elsif no_params %>
-  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name);
+  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
   <% else %>
-  SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>);
+  SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
   <% end %>
 $$ LANGUAGE plproxy;

client/renderer/templates/90_grant_execute.erb

@@ -1 +1 @@
-GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type %>) TO publicuser;
+GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;

client/sql/16_custom_types.sql

@@ -18,3 +18,20 @@ CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, deno
 CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric);
 
 CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean);
+
+
+-- For quotas and services configuration
+CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
+    'isolines',
+    'hires_geocoder',
+    'routing',
+    'observatory'
+);
+
+CREATE TYPE cdb_dataservices_client.service_quota_info AS (
+    service cdb_dataservices_client.service_type,
+    monthly_quota NUMERIC,
+    used_quota NUMERIC,
+    soft_limit BOOLEAN,
+    provider TEXT
+);

server/extension/sql/200_quotas.sql

@@ -0,0 +1,105 @@
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_type') THEN
+    CREATE TYPE cdb_dataservices_server.service_type AS ENUM (
+      'isolines',
+      'hires_geocoder',
+      'routing',
+      'observatory'
+    );
+  END IF;
+END $$;
+
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'service_quota_info') THEN
+    CREATE TYPE cdb_dataservices_server.service_quota_info AS (
+      service cdb_dataservices_server.service_type,
+      monthly_quota NUMERIC,
+      used_quota NUMERIC,
+      soft_limit BOOLEAN,
+      provider TEXT
+    );
+  END IF;
+END $$;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_quota_info(
+  username TEXT,
+  orgname TEXT)
+RETURNS SETOF cdb_dataservices_server.service_quota_info AS $$
+  from cartodb_services.metrics.user import UserMetricsService
+  from datetime import date
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  today = date.today()
+  ret = []
+
+  #-- Isolines
+  service = 'isolines'
+  plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
+  user_service = UserMetricsService(user_isolines_config, redis_conn)
+
+  monthly_quota = user_isolines_config.isolines_quota
+  used_quota = user_service.used_quota(user_isolines_config.service_type, today)
+  soft_limit = user_isolines_config.soft_isolines_limit
+  provider = user_isolines_config.provider
+  ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
+
+  #-- Hires Geocoder
+  service = 'hires_geocoder'
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  user_service = UserMetricsService(user_geocoder_config, redis_conn)
+
+  monthly_quota = user_geocoder_config.geocoding_quota
+  used_quota = user_service.used_quota(user_geocoder_config.service_type, today)
+  soft_limit = user_geocoder_config.soft_geocoding_limit
+  provider = user_geocoder_config.provider
+  ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
+
+  #-- Routing
+  service = 'routing'
+  plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_routing_config = GD["user_routing_config_{0}".format(username)]
+  user_service = UserMetricsService(user_routing_config, redis_conn)
+
+  monthly_quota = user_routing_config.monthly_quota
+  used_quota = user_service.used_quota(user_routing_config.service_type, today)
+  soft_limit = user_routing_config.soft_limit
+  provider = user_routing_config.provider
+  ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
+
+  #-- Observatory
+  service = 'observatory'
+  plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_obs_config = GD["user_obs_config_{0}".format(username)]
+  user_service = UserMetricsService(user_obs_config, redis_conn)
+
+  monthly_quota = user_obs_config.monthly_quota
+  used_quota = user_service.used_quota(user_obs_config.service_type, today)
+  soft_limit = user_obs_config.soft_limit
+  provider = user_obs_config.provider
+  ret += [[service, monthly_quota, used_quota, soft_limit, provider]]
+
+  return ret
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_enough_quota(
+  username TEXT,
+  orgname TEXT,
+  service_ TEXT,
+  input_size NUMERIC)
+returns BOOLEAN AS $$
+  DECLARE
+    params cdb_dataservices_server.service_quota_info;
+  BEGIN
+    SELECT * INTO params
+      FROM cdb_dataservices_server.cdb_service_quota_info(username, orgname) AS p
+      WHERE p.service = service_::cdb_dataservices_server.service_type;
+    RETURN params.soft_limit OR ((params.used_quota + input_size) <= params.monthly_quota);
+  END
+$$ LANGUAGE plpgsql;

server/lib/python/cartodb_services/README.md

@@ -29,12 +29,12 @@ NOTE: a system installation is required at present because the library is meant
 
 
 ## Running the unit tests
-Just run `nosetests`
+Just run `nosetests test/`
 ```shell
-$ nosetests
-.................................................
+$ nosetests test/
+......................................................................................................
 ----------------------------------------------------------------------
-Ran 49 tests in 0.131s
+Ran 102 tests in 0.122s
 
 OK
 ```

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -94,7 +94,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig):
             self._soft_limit = False
         self._monthly_quota = 0
         if self.QUOTA_KEY in self._redis_config:
-            self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
+            self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
         self._connection_str = self._db_config.data_observatory_connection_str
 
     @property
@@ -118,7 +118,7 @@ class ObservatoryConfig(DataObservatoryConfig):
             self._soft_limit = False
         self._monthly_quota = 0
         if self.QUOTA_KEY in self._redis_config:
-            self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
+            self._monthly_quota = int(self._redis_config[self.QUOTA_KEY])
         self._connection_str = self._db_config.data_observatory_connection_str
 
     @property
@@ -218,7 +218,7 @@ class IsolinesRoutingConfig(ServiceConfig):
         self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
         if self._isolines_provider == self.HEREMAPS_PROVIDER:
-            self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
+            self._isolines_quota = int(filtered_config[self.QUOTA_KEY])
             self._heremaps_app_id = db_config.heremaps_isolines_app_id
             self._heremaps_app_code = db_config.heremaps_isolines_app_code
             if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
@@ -361,7 +361,7 @@ class GeocoderConfig(ServiceConfig):
             self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER].lower()
         else:
             self._geocoder_provider = self.DEFAULT_PROVIDER
-        self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
+        self._geocoding_quota = int(filtered_config[self.QUOTA_KEY])
         self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
         if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
             self._soft_geocoding_limit = True

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -9,6 +9,7 @@ class UserMetricsService:
     SERVICE_GEOCODER_CACHE = 'geocoder_cache'
     SERVICE_HERE_ISOLINES = 'here_isolines'
     SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
+    SERVICE_OBSERVATORY = 'obs_general'
     DAY_OF_MONTH_ZERO_PADDED = '%d'
 
     def __init__(self, user_geocoder_config, redis_connection):
@@ -22,6 +23,8 @@ class UserMetricsService:
             return self.__used_isolines_quota(service_type, date)
         elif service_type == self.SERVICE_MAPZEN_ROUTING:
             return self.__used_routing_quota(service_type, date)
+        elif service_type == self.SERVICE_OBSERVATORY:
+            return self.__used_observatory_quota(service_type, date)
         else:
             return self.__used_geocoding_quota(service_type, date)
 
@@ -72,6 +75,19 @@ class UserMetricsService:
 
         return current_use
 
+    def __used_observatory_quota(self, service_type, date):
+        date_from, date_to = self.__current_billing_cycle()
+        current_use = 0
+        success_responses = self.get_metrics(service_type,
+                                             'success_responses', date_from,
+                                             date_to)
+        empty_responses = self.get_metrics(service_type,
+                                           'empty_responses', date_from,
+                                           date_to)
+        current_use += (success_responses + empty_responses)
+
+        return current_use
+
     def increment_service_use(self, service_type, metric, date=date.today(),
                               amount=1):
         """ Increment the services uses in monthly and daily basis"""
@@ -88,11 +104,11 @@ class UserMetricsService:
             redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
                                                      service, metric, date)
             score = self._redis_connection.zscore(redis_prefix, date.day)
-            aggregated_metric += score if score else 0
+            aggregated_metric += int(score) if score else 0
             zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
             if str(date.day) != zero_padded_day:
                 score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
-                aggregated_metric += score if score else 0
+                aggregated_metric += int(score) if score else 0
 
         return aggregated_metric
 

server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py

@@ -92,7 +92,7 @@ class MapzenGeocoderConfigBuilder(object):
         geocoding_quota = mapzen_server_conf['geocoder']['monthly_quota']
         mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
 
-        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit')
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
 
         cost_per_hit=0