diff --git a/server/extension/Makefile b/server/extension/Makefile index ca57d6e..f4500a5 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.6.2--0.7.0.sql \ - cdb_dataservices_server--0.7.0--0.6.2.sql + cdb_dataservices_server--0.7.0--0.7.1.sql \ + cdb_dataservices_server--0.7.1--0.7.0.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql b/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql new file mode 100644 index 0000000..0c4a66d --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql @@ -0,0 +1,94 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.1'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_data_observatory_config(text, text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_obs_snapshot_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import ObservatorySnapshotConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql b/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql new file mode 100644 index 0000000..c8b04ed --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql @@ -0,0 +1,94 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.0'" to load this file. \quit +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config(text, text); +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_data_observatory_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import DataObservatoryConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = data_observatory_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + time_span TEXT DEFAULT '2009 - 2013', + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e) + plpy.notice(traceback.format_exception(type_, value_, traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( + username TEXT, + orgname TEXT, + geom geometry(Geometry, 4326), + geometry_level TEXT DEFAULT '"us.census.tiger".block_group') +RETURNS json AS $$ + from cartodb_services.metrics import QuotaService + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + + quota_service = QuotaService(user_data_observatory_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reached the limit of your quota') + + try: + obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 23be467..ad3e6d9 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.7.0' +default_version = '0.7.1' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.6.2--0.7.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.2--0.7.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.2--0.7.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.2--0.7.0.sql diff --git a/server/extension/cdb_dataservices_server--0.7.0--0.6.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.0--0.6.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.0--0.6.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.0--0.6.2.sql diff --git a/server/extension/cdb_dataservices_server--0.7.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.7.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.7.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.7.0.sql diff --git a/server/extension/sql/110_data_observatory.sql b/server/extension/sql/110_data_observatory.sql index b77b101..842a166 100644 --- a/server/extension/sql/110_data_observatory.sql +++ b/server/extension/sql/110_data_observatory.sql @@ -10,10 +10,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - quota_service = QuotaService(user_data_observatory_config, redis_conn) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') @@ -48,10 +48,10 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - quota_service = QuotaService(user_data_observatory_config, redis_conn) + quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): plpy.error('You have reached the limit of your quota') diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 88b43c1..eb038fe 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -54,16 +54,16 @@ RETURNS boolean AS $$ return True $$ LANGUAGE plpythonu SECURITY DEFINER; -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text) RETURNS boolean AS $$ - cache_key = "user_data_observatory_config_{0}".format(username) + cache_key = "user_obs_snapshot_config_{0}".format(username) if cache_key in GD: return False else: - from cartodb_services.metrics import DataObservatoryConfig + from cartodb_services.metrics import ObservatorySnapshotConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname) - GD[cache_key] = data_observatory_config + obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = obs_snapshot_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index c357ade..0807d55 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 9b1b332..687f2c6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -32,19 +32,31 @@ class ServiceConfig(object): def organization(self): return self._orgname -class DataObservatoryConfig(ServiceConfig): +class ObservatorySnapshotConfig(ServiceConfig): + + SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit' + QUOTA_KEY = 'obs_snapshot_quota' PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_connection, db_conn, username, orgname=None): - super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, + super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn, username, orgname) - self._monthly_quota = self._db_config.data_observatory_monthly_quota self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) + if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_limit = True + else: + self._soft_limit = False + # Mixed config between db and redis. If we don't update all the users + # in redis, we could use the db value as default + if self.QUOTA_KEY in self._redis_config: + self._monthly_quota = float(self._redis_config[self.QUOTA_KEY]) + else: + self._monthly_quota = float(self._db_config.data_observatory_monthly_quota) @property def service_type(self): - return 'data_observatory' + return 'obs_snapshot' @property def monthly_quota(self): @@ -54,6 +66,10 @@ class DataObservatoryConfig(ServiceConfig): def period_end_date(self): return self._period_end_date + @property + def soft_limit(self): + return self._soft_limit + class RoutingConfig(ServiceConfig): @@ -200,28 +216,6 @@ class GeocoderConfig(ServiceConfig): self.__parse_config(filtered_config, self._db_config) self.__check_config(filtered_config) - def __get_user_config(self, username, orgname): - user_config = self._redis_connection.hgetall( - "rails:users:{0}".format(username)) - if not user_config: - raise ConfigException("""There is no user config available. Please check your configuration.'""") - else: - if orgname: - self.__get_organization_config(orgname, user_config) - - return user_config - - def __get_organization_config(self, orgname, user_config): - org_config = self._redis_connection.hgetall( - "rails:orgs:{0}".format(orgname)) - if not org_config: - raise ConfigException("""There is no organization config available. Please check your configuration.'""") - else: - user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] - user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] - user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] - user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] - def __check_config(self, filtered_config): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \ @@ -438,24 +432,24 @@ class ServicesRedisConfig: GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' QUOTA_KEY = 'geocoding_quota' ISOLINES_QUOTA_KEY = 'here_isolines_quota' + OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota' PERIOD_END_DATE = 'period_end_date' def __init__(self, redis_conn): self._redis_connection = redis_conn - def build(self, username, password): - return self.__get_user_config(username, password) + def build(self, username, orgname): + return self.__get_user_config(username, orgname) def __get_user_config(self, username, orgname): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") - else: - if orgname: - self.__get_organization_config(orgname, user_config) + elif orgname: + self.__get_organization_config(orgname, user_config) - return user_config + return user_config def __get_organization_config(self, orgname, user_config): org_config = self._redis_connection.hgetall( @@ -465,6 +459,8 @@ class ServicesRedisConfig: else: user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY] + if self.OBS_SNAPSHOT_QUOTA_KEY in org_config: + user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY] user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index b1d1137..2460082 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -73,9 +73,9 @@ class QuotaChecker: elif re.match('routing_mapzen', self._user_service_config.service_type) is not None: return self.__check_routing_quota() - elif re.match('data_observatory', + elif re.match('obs_snapshot', self._user_service_config.service_type) is not None: - return self.__check_data_observatory_quota() + return self.__check_obs_snapshot_quota() else: return False @@ -118,7 +118,7 @@ class QuotaChecker: else: return False - def __check_data_observatory_quota(self): + def __check_obs_snapshot_quota(self): user_quota = self._user_service_config.monthly_quota today = date.today() service_type = self._user_service_config.service_type diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 6e218e8..3dcba95 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.5.0', + version='0.5.1', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index b92dc31..8d980d2 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -1,5 +1,5 @@ import test_helper -from cartodb_services.metrics import GeocoderConfig, ConfigException +from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException from unittest import TestCase from nose.tools import assert_raises from mockredis import MockRedis @@ -32,10 +32,29 @@ class TestConfig(TestCase): assert geocoder_config.soft_geocoding_limit is False assert geocoder_config.period_end_date.date() == yesterday.date() + def test_should_return_config_for_obs_snapshot(self): + yesterday = datetime.today() - timedelta(days=1) + test_helper.build_redis_user_config(self.redis_conn, 'test_user', + do_quota=100, soft_do_limit=True, + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + 'test_user') + assert do_config.monthly_quota == 100 + assert do_config.soft_limit is True + assert do_config.period_end_date.date() == yesterday.date() + + def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): + yesterday = datetime.today() - timedelta(days=1) + test_helper.build_redis_user_config(self.redis_conn, 'test_user', + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + 'test_user') + assert do_config.monthly_quota == 100000 + assert do_config.soft_limit is False + assert do_config.period_end_date.date() == yesterday.date() + def test_should_raise_exception_when_missing_parameters(self): plpy_mock = test_helper.build_plpy_mock(empty=True) test_helper.build_redis_user_config(self.redis_conn, 'test_user') - assert_raises(ConfigException, - GeocoderConfig, - self.redis_conn, plpy_mock, 'test_user', - None) + assert_raises(ConfigException, GeocoderConfig, self.redis_conn, + plpy_mock, 'test_user', None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 000d260..60c85b1 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -4,6 +4,7 @@ from mock import Mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, service="heremaps", isolines_quota=0, + do_quota=None, soft_do_limit=None, end_date=datetime.today()): user_redis_name = "rails:users:{0}".format(username) redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit) @@ -11,23 +12,31 @@ def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, redis_conn.hset(user_redis_name, 'here_isolines_quota', isolines_quota) redis_conn.hset(user_redis_name, 'geocoder_type', service) redis_conn.hset(user_redis_name, 'period_end_date', end_date) + if do_quota: + redis_conn.hset(user_redis_name, 'obs_snapshot_quota', do_quota) + if soft_do_limit: + redis_conn.hset(user_redis_name, 'soft_obs_snapshot_limit', + soft_do_limit) redis_conn.hset(user_redis_name, 'google_maps_client_id', '') redis_conn.hset(user_redis_name, 'google_maps_api_key', '') -def build_redis_org_config(redis_conn, orgname, quota=100, isolines_quota=0, +def build_redis_org_config(redis_conn, orgname, quota=100, service="heremaps", + isolines_quota=0, do_quota=None, end_date=datetime.today()): org_redis_name = "rails:orgs:{0}".format(orgname) redis_conn.hset(org_redis_name, 'geocoding_quota', quota) redis_conn.hset(org_redis_name, 'here_isolines_quota', isolines_quota) + if do_quota: + redis_conn.hset(org_redis_name, 'obs_snapshot_quota', do_quota) redis_conn.hset(org_redis_name, 'period_end_date', end_date) redis_conn.hset(org_redis_name, 'google_maps_client_id', '') redis_conn.hset(org_redis_name, 'google_maps_api_key', '') -def increment_geocoder_uses(redis_conn, username, orgname=None, - date=date.today(), service='geocoder_here', - metric='success_responses', amount=20): +def increment_service_uses(redis_conn, username, orgname=None, + date=date.today(), service='geocoder_here', + metric='success_responses', amount=20): prefix = 'org' if orgname else 'user' entity_name = orgname if orgname else username yearmonth = date.strftime('%Y%m') diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index bcdacf8..eca9d3e 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,7 +1,7 @@ import test_helper from mockredis import MockRedis from cartodb_services.metrics import QuotaService -from cartodb_services.metrics import GeocoderConfig, RoutingConfig, DataObservatoryConfig +from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig from unittest import TestCase from nose.tools import assert_raises from datetime import datetime, date @@ -28,40 +28,40 @@ class TestQuotaService(TestCase): def test_should_return_true_if_user_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user') + test_helper.increment_service_uses(self.redis_conn, 'test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - orgname='test_org') + test_helper.increment_service_uses(self.redis_conn, 'test_user', + orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - amount=300) + test_helper.increment_service_uses(self.redis_conn, 'test_user', + amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - orgname='test_org', amount=400) + test_helper.increment_service_uses(self.redis_conn, 'test_user', + orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - amount=300) + test_helper.increment_service_uses(self.redis_conn, 'test_user', + amount=300) assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org', soft_limit=True) - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + test_helper.increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is True @@ -109,29 +109,34 @@ class TestQuotaService(TestCase): qs.increment_success_service_use(amount=1500000) assert qs.check_user_quota() is False - def test_should_check_user_data_observatory_quota_correctly(self): - qs = self.__build_data_observatory_quota_service('test_user') + def test_should_check_user_obs_snapshot_quota_correctly(self): + qs = self.__build_obs_snapshot_quota_service('test_user') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) assert qs.check_user_quota() is False - def test_should_check_org_data_observatory_quota_correctly(self): - qs = self.__build_data_observatory_quota_service('test_user', orgname='testorg') + def test_should_check_org_obs_snapshot_quota_correctly(self): + qs = self.__build_obs_snapshot_quota_service('test_user', + orgname='testorg') qs.increment_success_service_use() assert qs.check_user_quota() is True qs.increment_success_service_use(amount=100000) assert qs.check_user_quota() is False def __prepare_quota_service(self, username, quota, service, orgname, - soft_limit, end_date): + soft_limit, do_quota, soft_do_limit, end_date): test_helper.build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, + soft_do_limit=soft_do_limit, + do_quota=do_quota, end_date=end_date) if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, - quota=quota, end_date=end_date) + quota=quota, service=service, + do_quota=do_quota, + end_date=end_date) self._plpy_mock = test_helper.build_plpy_mock() def __build_geocoder_quota_service(self, username, quota=100, @@ -139,27 +144,27 @@ class TestQuotaService(TestCase): soft_limit=False, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, end_date) + soft_limit, 0, False, end_date) geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) - def __build_routing_quota_service(self, username, quota=100, - service='routing_mapzen', orgname=None, - soft_limit=False, - end_date=datetime.today()): + def __build_routing_quota_service(self, username, service='routing_mapzen', + orgname=None, soft_limit=False, + quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, end_date) + soft_limit, 0, False, end_date) routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) - def __build_data_observatory_quota_service(self, username, quota=100, - service='data_observatory', orgname=None, + def __build_obs_snapshot_quota_service(self, username, quota=100, + service='obs_snapshot', + orgname=None, soft_limit=False, end_date=datetime.today()): - self.__prepare_quota_service(username, quota, service, orgname, - soft_limit, end_date) - do_config = DataObservatoryConfig(self.redis_conn, self._plpy_mock, + self.__prepare_quota_service(username, 0, service, orgname, False, + quota, soft_limit, end_date) + do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 7f0d1fc..b173d31 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -18,15 +18,15 @@ class TestUserService(TestCase): def test_user_used_quota_for_a_day(self): us = self.__build_user_service('test_user') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - amount=400) + test_helper.increment_service_uses(self.redis_conn, 'test_user', + amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_org_used_quota_for_a_day(self): us = self.__build_user_service('test_user', orgname='test_org') - test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', - orgname='test_org', - amount=400) + test_helper.increment_service_uses(self.redis_conn, 'test_user', + orgname='test_org', + amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_user_not_amount_in_used_quota_for_month_should_be_0(self):