diff --git a/README.md b/README.md index 59c5d74..a2b74dd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Steps to deploy a new Data Services API version : ``` # in dataservices-api repo root path: - cd dataservices-api cd client && sudo make install cd - cd server/extension && sudo make install @@ -53,7 +52,7 @@ Steps to deploy a new Data Services API version : ``` # in dataservices-api repo root path: - cd server/lib/python/cartodb_services && pip install -r requirements.txt —upgrade + cd server/lib/python/cartodb_services && pip install -r requirements.txt && sudo pip install . --upgrade ``` - install extensions in user database @@ -66,7 +65,6 @@ Steps to deploy a new Data Services API version : create extension cdb_dataservices_client; ``` - ### Server configuration Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`. diff --git a/client/renderer/interface.yaml b/client/renderer/interface.yaml index fa7c257..7211802 100644 --- a/client/renderer/interface.yaml +++ b/client/renderer/interface.yaml @@ -421,3 +421,29 @@ params: - { name: service, type: TEXT } - { name: input_size, type: NUMERIC } + +- name: cdb_service_get_rate_limit + return_type: json + params: + - { name: service, type: "text" } + +- name: cdb_service_set_user_rate_limit + superuser: true + return_type: void + params: + - { name: service, type: "text" } + - { name: rate_limit, type: json } + +- name: cdb_service_set_org_rate_limit + superuser: true + return_type: void + params: + - { name: service, type: "text" } + - { name: rate_limit, type: json } + +- name: cdb_service_set_server_rate_limit + superuser: true + return_type: void + params: + - { name: service, type: "text" } + - { name: rate_limit, type: json } diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index 2516a09..4ad22d0 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -44,16 +44,29 @@ class SqlTemplateRenderer @function_signature['geocoder_config_key'] end - def params - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"} + def parameters_info(with_user_org) + parameters = [] + if with_user_org + parameters << { 'name' => 'username', 'type' => 'text' } + parameters << { 'name' => 'orgname', 'type' => 'text' } + end + parameters + @function_signature['params'].reject(&:empty?) end - def params_with_type - @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" } + def user_org_declaration() + "username text;\n orgname text;" unless superuser_function? end - def params_with_type_and_default - parameters = @function_signature['params'].reject(&:empty?).map do |p| + def params(with_user_org = superuser_function?) + parameters_info(with_user_org).map { |p| p['name'].to_s } + end + + def params_with_type(with_user_org = superuser_function?) + parameters_info(with_user_org).map { |p| "#{p['name']} #{p['type']}" } + end + + def params_with_type_and_default(with_user_org = superuser_function?) + parameters = parameters_info(with_user_org).map do |p| if not p['default'].nil? "#{p['name']} #{p['type']} DEFAULT #{p['default']}" else @@ -62,6 +75,49 @@ class SqlTemplateRenderer end return parameters end + + def superuser_function? + !!@function_signature['superuser'] + end + + def void_return_type? + return_type.downcase == 'void' + end + + def return_declaration + "ret #{return_type};" unless void_return_type? || multi_row + end + + def return_statement(&block) + if block + erb_out = block.binding.eval('_erbout') + + if multi_row + erb_out << 'RETURN QUERY SELECT * FROM ' + elsif multi_field + erb_out << 'SELECT * FROM ' + elsif void_return_type? + erb_out << 'PERFORM ' + else + erb_out << 'SELECT ' + end + yield + if multi_row || void_return_type? + erb_out << ';' + else + erb_out << ' INTO ret;' + end + if !multi_row && !void_return_type? + erb_out << ' RETURN ret;' + end + else + if !multi_row && !void_return_type? + ' RETURN ret;' + end + end + end + + end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 88db53d..abd6d05 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -7,9 +7,8 @@ CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>) RETURNS <%= return_type %> AS $$ DECLARE - <% if not multi_row %>ret <%= return_type %>;<% end %> - username text; - orgname text; + <%= return_declaration if not multi_row %> + <%= user_org_declaration %> BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; @@ -19,15 +18,7 @@ BEGIN IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - <% if multi_row %> - RETURN QUERY - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>); - <% elsif multi_field %> - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; - RETURN ret; - <% else %> - SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; - RETURN ret; - <% end %> + + <% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %> END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; diff --git a/client/renderer/templates/25_exception_safe_private_functions.erb b/client/renderer/templates/25_exception_safe_private_functions.erb index 2dde980..b0d2921 100644 --- a/client/renderer/templates/25_exception_safe_private_functions.erb +++ b/client/renderer/templates/25_exception_safe_private_functions.erb @@ -5,9 +5,8 @@ CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>) RETURNS <%= return_type %> AS $$ DECLARE - <% if not multi_row %>ret <%= return_type %>;<% end %> - username text; - orgname text; + <%= return_declaration %> + <%= user_org_declaration %> _returned_sqlstate TEXT; _message_text TEXT; _pg_exception_context TEXT; @@ -21,41 +20,16 @@ BEGIN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - <% if multi_row %> - BEGIN - RETURN QUERY - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>); - EXCEPTION - WHEN OTHERS THEN + + BEGIN + <% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %> + EXCEPTION + WHEN OTHERS THEN GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, _message_text = MESSAGE_TEXT, _pg_exception_context = PG_EXCEPTION_CONTEXT; RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; - END; - <% elsif multi_field %> - BEGIN - SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; - RETURN ret; - EXCEPTION - WHEN OTHERS THEN - GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, - _message_text = MESSAGE_TEXT, - _pg_exception_context = PG_EXCEPTION_CONTEXT; - RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; - RETURN ret; - END; - <% else %> - BEGIN - SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret; - RETURN ret; - EXCEPTION - WHEN OTHERS THEN - GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE, - _message_text = MESSAGE_TEXT, - _pg_exception_context = PG_EXCEPTION_CONTEXT; - RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context; - RETURN ret; - END; - <% end %> + <%= return_statement %> + END; END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; diff --git a/client/renderer/templates/30_plproxy_functions.erb b/client/renderer/templates/30_plproxy_functions.erb index 2d5dfe3..7a7a230 100644 --- a/client/renderer/templates/30_plproxy_functions.erb +++ b/client/renderer/templates/30_plproxy_functions.erb @@ -1,9 +1,9 @@ -CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>) +CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= params_with_type_and_default(_with_user_org=true).join(', ') %>) RETURNS <%= return_type %> AS $$ CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str(); <% if multi_field %> - SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); + SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>); <% else %> - SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>); + SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>); <% end %> $$ LANGUAGE plproxy; diff --git a/client/renderer/templates/90_grant_execute.erb b/client/renderer/templates/90_grant_execute.erb index b6c797e..1b8d7c8 100644 --- a/client/renderer/templates/90_grant_execute.erb +++ b/client/renderer/templates/90_grant_execute.erb @@ -1,2 +1,4 @@ +<% unless superuser_function? %> GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser; -GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser; +GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %> ) TO publicuser; +<% end %> \ No newline at end of file diff --git a/doc/rate_limits.md b/doc/rate_limits.md new file mode 100644 index 0000000..333a8f3 --- /dev/null +++ b/doc/rate_limits.md @@ -0,0 +1,195 @@ +# Rate limits + +Services can be rate-limited. (currently only gecoding is limited) + +The rate limits configuration can be established at server, organization or user levels, the latter having precedence over the earlier. + +The default configuration (a null or empty configuration) doesn't impose any limits. + +The configuration consist of a JSON object with two attributes: + +* `period`: the rate-limiting period, in seconds. +* `limit`: the maximum number of request in the established period. + +If a service request exceeds the configured rate limits +(i.e. if more than `limit` calls are performe in a fixed interval of +duration `period` seconds) the call will fail with an "Rate limit exceeded" error. + +## Server-side interface + +There's a server-side SQL interface to query or change the configuration. + +### cdb_dataservices_server.cdb_service_get_rate_limit(username, orgname, service) + +This function returns the rate limit configuration for a given user and service. + +#### Returns + +The result is a JSON object with the configuration (`period` and `limit` attributes as explained above). + +### cdb_dataservices_server.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit) + +This function sets the rate limit configuration for the user. This overrides any other configuration. + +The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +### cdb_dataservices_server.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit) + +This function sets the rate limit configuration for the organization. +This overrides server level configuration and is overriden by user configuration if present. + +The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +### cdb_dataservices_server.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit) + +This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration. + +The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +## Client-side interface + +For convenience there's also a client-side interface (in the client dataservices-api extension), consisting +of public functions to get the current configuration and privileged functions to change it. + +### Public functions + +These functions are accesible to non-privileged roles, and should only be executed +using the role corresponding to a CARTO user, since that will determine the +user and organization to which the rate limits configuration applies. + +### cdb_dataservices_client.cdb_service_get_rate_limit(service) + +This function returns the rate limit configuration in effect for the specified service +and the user corresponding to the role which makes the calls. The effective configuration +may come from any of the configuration levels (server/organization/user); only the +existing configuration with most precedence is returned. + +#### Returns + +The result is a JSON object with the configuration (`period` and `limit` attributes as explained above). + +#### Example: + +``` +SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoding'); + + cdb_service_get_rate_limit +--------------------------------- + {"limit": 1000, "period": 86400} +(1 row) +``` + + +### Privileged (superuser) functions + +Thes functions are not accessible by regular user roles, and the user and organization names must be provided as parameters. + +### cdb_dataservices_client.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit) + +This function sets the rate limit configuration for the user. This overrides any other configuration. + +The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +#### Example + +This will configure the geocoding service rate limit for user `myusername`, a non-organization user. +The limit will be set at 1000 requests per day. Since the user doesn't belong to any organization, +`NULL` will be passed to the organization argument; otherwise the name of the user's organization should +be provided. + +``` +SELECT cdb_dataservices_client.cdb_service_set_user_rate_limit( + 'myusername', + NULL, + 'geocoding', + '{"limit":1000,"period":86400}' +); + + cdb_service_set_user_rate_limit +--------------------------------- + +(1 row) +``` + +### cdb_dataservices_client.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit) + +This function sets the rate limit configuration for the organization. +This overrides server level configuration and is overriden by user configuration if present. + +The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +#### Example + +This will configure the geocoding service rate limit for the `myorg` organization. +The limit will be set at 100 requests per hour. +Note that even we're setting the default configuration for the whole organization, +the name of a user of the organization must be provided for technical reasons. + +``` +SELECT cdb_dataservices_client.cdb_service_set_org_rate_limit( + 'myorgadmin', + 'myorg', + 'geocoding', + '{"limit":100,"period":3600}' +); + + + cdb_service_set_org_rate_limit +--------------------------------- + +(1 row) +``` + +### cdb_dataservices_client.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit) + +This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration. + +The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`. + +#### Returns + +This functions doesn't return any value. + +#### Example + +This will configure the default geocoding service rate limit for all users +accesing the data-services server. +The limit will be set at 10000 requests per month. +Note that even we're setting the default configuration for the server, +the name of a user and the name of the corresponding organization (or NULL) +must be provided for technical reasons. + +``` +SELECT cdb_dataservices_client.cdb_service_set_server_rate_limit( + 'myorgadmin', + 'myorg', + 'geocoding', + '{"limit":10000,"period":108000}' +); + + + cdb_service_set_server_rate_limit +--------------------------------- + +(1 row) +``` diff --git a/server/extension/sql/15_config_helper.sql b/server/extension/sql/15_config_helper.sql index 9d83ebd..2c5acf7 100644 --- a/server/extension/sql/15_config_helper.sql +++ b/server/extension/sql/15_config_helper.sql @@ -16,6 +16,24 @@ RETURNS JSON AS $$ SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key; $$ LANGUAGE SQL STABLE SECURITY DEFINER; +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON) + RETURNS void AS $$ +BEGIN + PERFORM cartodb.CDB_Conf_RemoveConf(key); + EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value; +END +$$ LANGUAGE PLPGSQL VOLATILE; + +CREATE OR REPLACE +FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text) + RETURNS void AS $$ +BEGIN + EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key; +END +$$ LANGUAGE PLPGSQL VOLATILE; + + CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL) RETURNS boolean AS $$ cache_key = "user_geocoder_config_{0}".format(username) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index a8c9067..ffed77c 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -72,23 +72,15 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager from cartodb_services.here import HereMapsGeocoder - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - # -- Check the quota - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + service_manager.assert_within_limits() try: - geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params) + geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -96,46 +88,41 @@ RETURNS Geometry AS $$ point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return None except BaseException as e: import sys - quota_service.increment_failed_service_use() + service_manager.quota_service.increment_failed_service_use() logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to geocode street point using here maps') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.tools import LegacyServiceManager from cartodb_services.google import GoogleMapsGeocoder - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger,LoggerConfig - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) + service_manager = LegacyServiceManager('geocoder', username, orgname, GD) + service_manager.assert_within_limits(quota=False) try: - geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger) + geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return None except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to geocode street point using google maps') finally: quota_service.increment_total_service_use() @@ -143,38 +130,19 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ - import cartodb_services - cartodb_services.init(plpy, GD) + from cartodb_services.tools import ServiceManager from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 - from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger - from cartodb_services.refactor.tools.logger import LoggerConfigBuilder from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder - from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder - from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory - from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory - from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory - from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory - server_config_backend = ServerConfigBackendFactory().get() - environment = ServerEnvironmentBuilder(server_config_backend).get() - user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get() - org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get() + import cartodb_services + cartodb_services.init(plpy, GD) - logger_config = LoggerConfigBuilder(environment, server_config_backend).get() - logger = Logger(logger_config) - - mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get() - - redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get() - - quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection) - if not quota_service.check_user_quota(): - raise Exception('You have reached the limit of your quota') + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname) + service_manager.assert_within_limits() try: - geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) + geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params) country_iso3 = None if country: country_iso3 = country_to_iso3(country) @@ -182,18 +150,18 @@ RETURNS Geometry AS $$ state_province=state_province, country=country_iso3, search_type='address') if coordinates: - quota_service.increment_success_service_use() + service_manager.quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] return point['st_setsrid'] else: - quota_service.increment_empty_service_use() + service_manager.quota_service.increment_empty_service_use() return None except BaseException as e: import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + service_manager.quota_service.increment_failed_service_use() + service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) raise Exception('Error trying to geocode street point using mapzen') finally: - quota_service.increment_total_service_use() + service_manager.quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/210_rates.sql b/server/extension/sql/210_rates.sql new file mode 100644 index 0000000..058c766 --- /dev/null +++ b/server/extension/sql/210_rates.sql @@ -0,0 +1,91 @@ + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit( + username TEXT, + orgname TEXT, + service TEXT) +RETURNS JSON AS $$ + import json + from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + + import cartodb_services + cartodb_services.init(plpy, GD) + + service_config = ServiceConfiguration(service, username, orgname) + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + if rate_limit_config.is_limited(): + return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period}) + else: + return None +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_user_rate_limits(config) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_org_rate_limits(config) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit( + username TEXT, + orgname TEXT, + service TEXT, + rate_limit_json JSON) +RETURNS VOID AS $$ + import json + from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter + + import cartodb_services + cartodb_services.init(plpy, GD) + + config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + limit = rate_limit.get('limit', None) + period = rate_limit.get('period', None) + else: + limit = None + period = None + config = RateLimitsConfig(service=service, username=username, limit=limit, period=period) + config_setter.set_server_rate_limits(config) +$$ LANGUAGE plpythonu; diff --git a/server/extension/test/expected/210_rates_test.out b/server/extension/test/expected/210_rates_test.out new file mode 100644 index 0000000..f79cb33 --- /dev/null +++ b/server/extension/test/expected/210_rates_test.out @@ -0,0 +1,44 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_get_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_user_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_org_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_server_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/sql/210_rates_test.sql b/server/extension/test/sql/210_rates_test.sql new file mode 100644 index 0000000..752cdf4 --- /dev/null +++ b/server/extension/test/sql/210_rates_test.sql @@ -0,0 +1,27 @@ +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_get_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_user_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_org_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_service_set_server_rate_limit' + AND oidvectortypes(p.proargtypes) = 'text, text, text, json'); diff --git a/server/lib/python/cartodb_services/cartodb_services/config/__init__.py b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py new file mode 100644 index 0000000..6a98689 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/config/__init__.py @@ -0,0 +1,3 @@ +from service_configuration import ServiceConfiguration +from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter +from legacy_rate_limits import RateLimitsConfigLegacyBuilder diff --git a/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py new file mode 100644 index 0000000..e50da06 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py @@ -0,0 +1,46 @@ +import json +from rate_limits import RateLimitsConfig + +class RateLimitsConfigLegacyBuilder(object): + """ + Build a RateLimitsConfig object using the *legacy* configuration classes + """ + + def __init__(self, redis_connection, db_conn, service, username, orgname): + self._service = service + self._username = username + self._orgname = orgname + self._redis_connection = redis_connection + self._db_conn = db_conn + + def get(self): + rate_limit = self.__get_rate_limit() + return RateLimitsConfig(self._service, + self._username, + rate_limit.get('limit', None), + rate_limit.get('period', None)) + + def __get_rate_limit(self): + rate_limit = {} + rate_limit_key = "{0}_rate_limit".format(self._service) + user_key = "rails:users:{0}".format(self._username) + rate_limit_json = self.__get_redis_config(user_key, rate_limit_key) + if not rate_limit_json and self._orgname: + org_key = "rails:orgs:{0}".format(self._orgname) + rate_limit_json = self.__get_redis_config(org_key, rate_limit_key) + if rate_limit_json: + rate_limit = json.loads(rate_limit_json) + else: + conf_key = 'rate_limits' + sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(conf_key) + try: + conf = self._db_conn.execute(sql, 1)[0]['conf'] + except Exception: + conf = None + if conf: + rate_limit = json.loads(conf).get(self._service) + return rate_limit or {} + + def __get_redis_config(self, basekey, param): + config = self._redis_connection.hgetall(basekey) + return config and config.get(param) diff --git a/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py new file mode 100644 index 0000000..98f6a47 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py @@ -0,0 +1,113 @@ +import json + +from service_configuration import ServiceConfiguration + +class RateLimitsConfig(object): + """ + Value object that represents the configuration needed to rate-limit services + """ + + def __init__(self, + service, + username, + limit, + period): + self._service = service + self._username = username + self._limit = limit and int(limit) + self._period = period and int(period) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + # service this limit applies to + @property + def service(self): + return self._service + + # user this limit applies to + @property + def username(self): + return self._username + + # rate period in seconds + @property + def period(self): + return self._period + + # rate limit in seconds + @property + def limit(self): + return self._limit + + def is_limited(self): + return self._limit and self._limit > 0 and self._period and self._period > 0 + +class RateLimitsConfigBuilder(object): + """ + Build a rate limits configuration obtaining the parameters + from the user/org/server configuration. + """ + + def __init__(self, server_conf, user_conf, org_conf, service, username, orgname): + self._server_conf = server_conf + self._user_conf = user_conf + self._org_conf = org_conf + self._service = service + self._username = username + self._orgname = orgname + + def get(self): + # Order of precedence is user_conf, org_conf, server_conf + + rate_limit_key = "{0}_rate_limit".format(self._service) + + rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None) + if (rate_limit_json): + rate_limit = rate_limit_json and json.loads(rate_limit_json) + else: + rate_limit = self._server_conf.get('rate_limits', {}).get(self._service, {}) + + return RateLimitsConfig(self._service, + self._username, + rate_limit.get('limit', None), + rate_limit.get('period', None)) + + +class RateLimitsConfigSetter(object): + + def __init__(self, service, username, orgname=None): + self._service = service + self._service_config = ServiceConfiguration(service, username, orgname) + + def set_user_rate_limits(self, rate_limits_config): + # Note we allow copying a config from another user/service, so we + # ignore rate_limits:config.service and rate_limits:config.username + rate_limit_key = "{0}_rate_limit".format(self._service) + if rate_limits_config.is_limited(): + rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + rate_limit_json = json.dumps(rate_limit) + self._service_config.user.set(rate_limit_key, rate_limit_json) + else: + self._service_config.user.remove(rate_limit_key) + + def set_org_rate_limits(self, rate_limits_config): + rate_limit_key = "{0}_rate_limit".format(self._service) + if rate_limits_config.is_limited(): + rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + rate_limit_json = json.dumps(rate_limit) + self._service_config.org.set(rate_limit_key, rate_limit_json) + else: + self._service_config.org.remove(rate_limit_key) + + def set_server_rate_limits(self, rate_limits_config): + rate_limits = self._service_config.server.get('rate_limits', {}) + if rate_limits_config.is_limited(): + rate_limits[self._service] = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period} + else: + rate_limits.pop(self._service, None) + if rate_limits: + self._service_config.server.set('rate_limits', rate_limits) + else: + self._service_config.server.remove('rate_limits') + diff --git a/server/lib/python/cartodb_services/cartodb_services/config/service_configuration.py b/server/lib/python/cartodb_services/cartodb_services/config/service_configuration.py new file mode 100644 index 0000000..468a915 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/config/service_configuration.py @@ -0,0 +1,36 @@ +from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder +from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory +from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory +from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory + +class ServiceConfiguration(object): + """ + This class instantiates configuration backend objects for all the configuration levels of a service: + * environment + * server + * organization + * user + The configuration backends allow retrieval and modification of configuration parameters. + """ + + def __init__(self, service, username, orgname): + self._server_config_backend = ServerConfigBackendFactory().get() + self._environment = ServerEnvironmentBuilder(self._server_config_backend ).get() + self._user_config_backend = UserConfigBackendFactory(username, self._environment, self._server_config_backend ).get() + self._org_config_backend = OrgConfigBackendFactory(orgname, self._environment, self._server_config_backend ).get() + + @property + def environment(self): + return self._environment + + @property + def server(self): + return self._server_config_backend + + @property + def user(self): + return self._user_config_backend + + @property + def org(self): + return self._org_config_backend \ No newline at end of file diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py index 46dcd42..cb69b6f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py @@ -6,6 +6,6 @@ class ConfigBackendInterface(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod - def get(self, key): + def get(self, key, default=None): """Return a value based on the key supplied from some storage""" pass diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py index 0661626..148b970 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py @@ -5,8 +5,8 @@ class InMemoryConfigStorage(ConfigBackendInterface): def __init__(self, config_hash={}): self._config_hash = config_hash - def get(self, key): + def get(self, key, default=None): try: return self._config_hash[key] except KeyError: - return None + return default diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py index b9c11e6..c080e13 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py @@ -2,5 +2,5 @@ from ..core.interfaces import ConfigBackendInterface class NullConfigStorage(ConfigBackendInterface): - def get(self, key): - return None + def get(self, key, default=None): + return default diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py index cdb2e57..10dd21e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -9,11 +9,19 @@ class RedisConfigStorage(ConfigBackendInterface): self._config_key = config_key self._data = None - def get(self, key): + def get(self, key, default=KeyError): if not self._data: self._data = self._connection.hgetall(self._config_key) - return self._data[key] + if (default == KeyError): + return self._data[key] + else: + return self._data.get(key, default) + def set(self, key, value): + self._connection.hset(self._config_key, key, value) + + def remove(self, key): + self._connection.hdel(self._config_key, key) class RedisUserConfigStorageBuilder(object): def __init__(self, redis_connection, username): diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py index 50137ca..67b6328 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py @@ -4,11 +4,28 @@ from ..core.interfaces import ConfigBackendInterface class InDbServerConfigStorage(ConfigBackendInterface): - def get(self, key): + def get(self, key, default=None): sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key) rows = cartodb_services.plpy.execute(sql, 1) - json_output = rows[0]['conf'] - if json_output: + json_output = None + try: + json_output = rows[0]['conf'] + except (IndexError, KeyError): + pass + if (json_output): return json.loads(json_output) else: - return None + if (default == KeyError): + raise KeyError + else: + return default + + def set(self, key, config): + json_config = json.dumps(config) + quoted_config = cartodb_services.plpy.quote_nullable(json_config) + sql = "SELECT cdb_dataservices_server.cdb_conf_setconf('{0}', {1})".format(key, quoted_config) + cartodb_services.plpy.execute(sql) + + def remove(self, key): + sql = "SELECT cdb_dataservices_server.cdb_conf_removeconf('{0}')".format(key) + cartodb_services.plpy.execute(sql) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index 7bea3e2..16072f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -2,3 +2,6 @@ from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine from log import Logger, LoggerConfig +from rate_limiter import RateLimiter +from service_manager import ServiceManager, RateLimitExceeded +from legacy_service_manager import LegacyServiceManager diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/tools/exceptions.py new file mode 100644 index 0000000..e69de29 diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/legacy_service_manager.py b/server/lib/python/cartodb_services/cartodb_services/tools/legacy_service_manager.py new file mode 100644 index 0000000..2ae700c --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/legacy_service_manager.py @@ -0,0 +1,23 @@ +from cartodb_services.metrics import QuotaService +from cartodb_services.tools import Logger,LoggerConfig +from cartodb_services.tools import RateLimiter +from cartodb_services.config import RateLimitsConfigLegacyBuilder +from cartodb_services.tools.service_manager import ServiceManagerBase +import plpy + +class LegacyServiceManager(ServiceManagerBase): + """ + This service manager relies on cached configuration (in gd) stored in *legacy* configuration objects + It's intended for use by the *legacy* configuration objects (in use prior to the configuration refactor). + """ + + def __init__(self, service, username, orgname, gd): + redis_conn = gd["redis_connection_{0}".format(username)]['redis_metrics_connection'] + self.config = gd["user_{0}_config_{1}".format(service, username)] + logger_config = gd["logger_config"] + self.logger = Logger(logger_config) + + self.quota_service = QuotaService(self.config, redis_conn) + + rate_limit_config = RateLimitsConfigLegacyBuilder(redis_conn, plpy, service=service, username=username, orgname=orgname).get() + self.rate_limiter = RateLimiter(rate_limit_config, redis_conn) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py new file mode 100644 index 0000000..3410519 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py @@ -0,0 +1,18 @@ +from rratelimit import Limiter + +class RateLimiter: + + def __init__(self, rate_limits_config, redis_connection): + self._config = rate_limits_config + self._limiter = None + if (self._config.is_limited()): + self._limiter = Limiter(redis_connection, + action=self._config.service, + limit=self._config.limit, + period=self._config.period) + + def check(self): + ok = True + if (self._limiter): + ok = self._limiter.checked_insert(self._config.username) + return ok diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py new file mode 100644 index 0000000..d493e40 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py @@ -0,0 +1,70 @@ +from cartodb_services.metrics import QuotaService +from cartodb_services.tools import Logger +from cartodb_services.tools import RateLimiter +from cartodb_services.refactor.tools.logger import LoggerConfigBuilder +from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory +from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder + +class RateLimitExceeded(Exception): + def __str__(self): + return repr('Rate limit exceeded') + +class ServiceManagerBase: + """ + A Service manager collects the configuration needed to use a service, + including thir-party services parameters. + + This abstract class serves as the base for concrete service manager classes; + derived class must provide and initialize attributes for ``config``, + ``quota_service``, ``logger`` and ``rate_limiter`` (which can be None + for no limits). + + It provides an `assert_within_limits` method to check quota and rate limits + which raises exceptions when limits are exceeded. + + It exposes properties containing: + + * ``config`` : a configuration object containing the configuration parameters for + a given service and provider. + * ``quota_service`` a QuotaService object to for quota accounting + * ``logger`` + + """ + + def assert_within_limits(self, quota=True, rate=True): + if rate and not self.rate_limiter.check(): + raise RateLimitExceeded() + if quota and not self.quota_service.check_user_quota(): + raise Exception('You have reached the limit of your quota') + + @property + def config(self): + return self.config + + @property + def quota_service(self): + return self.quota_service + + @property + def logger(self): + return self.logger +class ServiceManager(ServiceManagerBase): + """ + This service manager delegates the configuration parameter details, + and the policies about configuration precedence to a configuration-builder class. + It uses the refactored configuration classes. + """ + + def __init__(self, service, config_builder, username, orgname): + service_config = ServiceConfiguration(service, username, orgname) + + logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get() + self.logger = Logger(logger_config) + + self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get() + rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get() + + redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get() + + self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection) + self.quota_service = QuotaService(self.config, redis_metrics_connection) diff --git a/server/lib/python/cartodb_services/requirements.txt b/server/lib/python/cartodb_services/requirements.txt index 1b8b873..822d980 100644 --- a/server/lib/python/cartodb_services/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -4,7 +4,8 @@ python-dateutil==2.2 googlemaps==2.4.2 rollbar==0.13.2 # Dependency for googlemaps package -requests<=2.9.1 +requests==2.9.1 +rratelimit==0.0.4 # Test mock==1.3.0 diff --git a/server/lib/python/cartodb_services/test/mock_plpy.py b/server/lib/python/cartodb_services/test/mock_plpy.py index 5298a98..0fe067f 100644 --- a/server/lib/python/cartodb_services/test/mock_plpy.py +++ b/server/lib/python/cartodb_services/test/mock_plpy.py @@ -16,7 +16,7 @@ class MockPlPy: def __init__(self): self._reset() - def _reset(self): + def _reset(self, log_executed_queries=False): self.infos = [] self.notices = [] self.debugs = [] @@ -28,11 +28,30 @@ class MockPlPy: self.results = [] self.prepares = [] self.results = {} + self._log_executed_queries = log_executed_queries + self._logged_queries = [] def _define_result(self, query, result): pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) self.results[pattern] = result + def _executed_queries(self): + if self._log_executed_queries: + return self._logged_queries + else: + raise Exception('Executed queries logging is not active') + + def _has_executed_query(self, query): + pattern = re.compile(re.escape(query)) + for executed_query in self._executed_queries(): + if pattern.search(executed_query): + return True + return False + + def _start_logging_executed_queries(self): + self._logged_queries = [] + self._log_executed_queries = True + def notice(self, msg): self.notices.append(msg) @@ -47,7 +66,15 @@ class MockPlPy: return MockCursor(data) def execute(self, query, rows=1): + if self._log_executed_queries: + self._logged_queries.append(query) for pattern, result in self.results.iteritems(): if pattern.search(query): return result return [] + + def quote_nullable(self, value): + if value is None: + return 'NULL' + else: + return "'{0}'".format(value) diff --git a/server/lib/python/cartodb_services/test/test_ratelimitsconfig.py b/server/lib/python/cartodb_services/test/test_ratelimitsconfig.py new file mode 100644 index 0000000..5cb967c --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_ratelimitsconfig.py @@ -0,0 +1,124 @@ +from test_helper import * +from unittest import TestCase +from mock import Mock, MagicMock, patch +from nose.tools import assert_raises, assert_not_equal, assert_equal + +from datetime import datetime, date +from mockredis import MockRedis +import cartodb_services + +from cartodb_services.tools import ServiceManager, LegacyServiceManager + +from cartodb_services.metrics import GeocoderConfig +from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder +from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder +from cartodb_services.tools import RateLimitExceeded + +from cartodb_services.refactor.storage.redis_config import * +from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage +from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory +from cartodb_services.config import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter + +class TestRateLimitsConfig(TestCase): + + def setUp(self): + plpy_mock_config() + cartodb_services.init(plpy_mock, _GD={}) + self.username = 'test_user' + self.orgname = 'test_org' + self.redis_conn = MockRedis() + build_redis_user_config(self.redis_conn, self.username, 'geocoding') + build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen') + self.environment = 'production' + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + basic_server_conf = {"server_conf": {"environment": "testing"}, + "mapzen_conf": + {"geocoder": + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}} + }, "logger_conf": {}} + self.empty_server_config = InMemoryConfigStorage(basic_server_conf) + self.empty_redis_config = InMemoryConfigStorage({}) + self.user_config = RedisUserConfigStorageBuilder(self.redis_conn, self.username).get() + self.org_config = RedisOrgConfigStorageBuilder(self.redis_conn, self.orgname).get() + self.server_config = ServerConfigBackendFactory().get() + + def test_server_config(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + + # Write server level configuration + config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400) + config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname) + plpy_mock._start_logging_executed_queries() + config_setter.set_server_rate_limits(config) + assert plpy_mock._has_executed_query('cdb_conf_setconf(\'rate_limits\', \'{"geocoder": {"limit": 1234, "period": 86400}}\')') + + # Re-read configuration + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}]) + read_config = RateLimitsConfigBuilder( + server_conf=self.server_config, + user_conf=self.empty_redis_config, + org_conf=self.empty_redis_config, + service='geocoder', + username=self.username, + orgname=self.orgname + ).get() + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + assert_equal(read_config, config) + + def test_server_org_config(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + + server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400) + org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400) + config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname) + + # Write server level configuration + config_setter.set_server_rate_limits(server_config) + # Override with org level configuration + config_setter.set_org_rate_limits(org_config) + + # Re-read configuration + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}]) + read_config = RateLimitsConfigBuilder( + server_conf=self.server_config, + user_conf=self.empty_redis_config, + org_conf=self.org_config, + service='geocoder', + username=self.username, + orgname=self.orgname + ).get() + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + assert_equal(read_config, org_config) + + def test_server_org_user_config(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + + server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400) + org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400) + user_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1236, period=86400) + config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname) + + # Write server level configuration + config_setter.set_server_rate_limits(server_config) + # Override with org level configuration + config_setter.set_org_rate_limits(org_config) + # Override with user level configuration + config_setter.set_user_rate_limits(user_config) + + # Re-read configuration + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}]) + read_config = RateLimitsConfigBuilder( + server_conf=self.server_config, + user_conf=self.user_config, + org_conf=self.org_config, + service='geocoder', + username=self.username, + orgname=self.orgname + ).get() + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + assert_equal(read_config, user_config) diff --git a/server/lib/python/cartodb_services/test/test_servicemanager.py b/server/lib/python/cartodb_services/test/test_servicemanager.py new file mode 100644 index 0000000..2131815 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_servicemanager.py @@ -0,0 +1,218 @@ +from test_helper import * +from unittest import TestCase +from mock import Mock, MagicMock, patch +from nose.tools import assert_raises, assert_not_equal, assert_equal + +from datetime import datetime, date +from cartodb_services.tools import ServiceManager, LegacyServiceManager +from mockredis import MockRedis +import cartodb_services +from cartodb_services.metrics import GeocoderConfig +from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder +from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder +from cartodb_services.tools import RateLimitExceeded + +LUA_AVAILABLE_FOR_MOCKREDIS = False + +class MockRedisWithVersionInfo(MockRedis): + def info(self): + return {'redis_version': '3.0.2'} + +class TestServiceManager(TestCase): + + def setUp(self): + plpy_mock_config() + cartodb_services.init(plpy_mock, _GD={}) + self.username = 'test_user' + self.orgname = 'test_org' + self.redis_conn = MockRedisWithVersionInfo() + build_redis_user_config(self.redis_conn, self.username, 'geocoding') + build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen') + self.environment = 'production' + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}]) + + def check_rate_limit(self, service_manager, n, active=True): + if LUA_AVAILABLE_FOR_MOCKREDIS: + for _ in xrange(n): + service_manager.assert_within_limits() + if active: + with assert_raises(RateLimitExceeded): + service_manager.assert_within_limits() + else: + service_manager.assert_within_limits() + else: + # rratelimit doesn't work with MockRedis because it needs Lua support + # so, we'll simply perform some sanity check on the configuration of the rate limiter + if active: + assert_equal(service_manager.rate_limiter._config.is_limited(), True) + assert_equal(service_manager.rate_limiter._config.limit, n) + else: + assert not service_manager.rate_limiter._config.is_limited() + + def test_legacy_service_manager(self): + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + service_manager.assert_within_limits() + assert_equal(service_manager.config.service_type, 'geocoder_mapzen') + + def test_service_manager(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + service_manager.assert_within_limits() + assert_equal(service_manager.config.service_type, 'geocoder_mapzen') + + def test_no_rate_limit_by_default(self): + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3, False) + + def test_no_legacy_rate_limit_by_default(self): + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3, False) + + def test_legacy_server_rate_limit(self): + rate_limits = '{"geocoder":{"limit":"3","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}]) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_server_rate_limit(self): + rate_limits = '{"geocoder":{"limit":"3","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}]) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_user_rate_limit(self): + user_redis_name = "rails:users:{0}".format(self.username) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_legacy_user_rate_limit(self): + user_redis_name = "rails:users:{0}".format(self.username) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_org_rate_limit(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + + def test_legacy_org_rate_limit(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + + def test_user_rate_limit_precedence_over_org(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"1000","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + user_redis_name = "rails:users:{0}".format(self.username) + user_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_org_rate_limit_precedence_over_server(self): + server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}]) + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + with patch.object(RedisConnectionBuilder,'get') as get_fn: + get_fn.return_value = self.redis_conn + service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", []) + + def test_legacy_user_rate_limit_precedence_over_org(self): + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"1000","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + user_redis_name = "rails:users:{0}".format(self.username) + user_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit') + + def test_legacy_org_rate_limit_precedence_over_server(self): + server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}' + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}]) + org_redis_name = "rails:orgs:{0}".format(self.orgname) + org_rate_limits = '{"limit":"3","period":3600}' + self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits) + config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname, 'mapzen') + config_cache = { + 'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn }, + 'user_geocoder_config_test_user' : config, + 'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment) + } + service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache) + self.check_rate_limit(service_manager, 3) + self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit') + plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])