Add new params at the end #361

Changes in TYPEs obs_meta_geometry and obs_meta_timespan

NEWS.md

@@ -1,3 +1,21 @@
+May 9th, 2017
+=============
+* Version `0.17.0` of the client and version `0.24.0` of the server
+    * Fixed some missing return values documented but not present.
+        *  `OBS_GetAvailableGeometries` now returns `geom_type`, `geom_extra` and `geom_tags` in addition to existing values.
+        *  `OBS_GetAvailableTimespans` now returns `timespan_type`, `timespan_extra`, `timespan_tags` in addition to existing values.
+
+March 28th, 2017
+================
+
+* Version 0.16.0 of the client, version 0.23.0 of the server and version 0.15.0 of the python library
+  * Added support for rate-limiting the geocoding services. See #365
+
+March 13th, 2017
+================
+* Version `0.14.1` of the python library:
+  * Clean up code that reads from non zero padded date keys #206
+
 March 8th, 2017
 ===============
 * Version 0.22.0 of the server and version 0.14.0 of the python library

README.md

@@ -23,47 +23,81 @@ Steps to deploy a new Data Services API version :
 
 ### Local install instructions
 
-- install data services geocoder extension
-
-    ```
-    git clone https://github.com/CartoDB/data-services.git
-    cd data-services/geocoder/extension
-    sudo make install
-    ```
-
-- install observatory extension
-
-    ```
-    git clone https://github.com/CartoDB/observatory-extension.git
-    cd observatory
-    sudo make install
-    ```
-
 - install server and client extensions
 
     ```
-    # in data-services repo root path:
+    # in your workspace root path
+    git clone https://github.com/CartoDB/dataservices-api.git
+    cd dataservices-api
     cd client && sudo make install
+    cd -
     cd server/extension && sudo make install
     ```
 
 - install python library
 
     ```
-    # in data-services repo root path:
-    cd server/lib/python/cartodb_services && sudo pip install . --upgrade
+    # in dataservices-api repo root path:
+    cd server/lib/python/cartodb_services && pip install -r requirements.txt && sudo pip install . --upgrade
     ```
 
-- install extensions in user database
+- Create a database to hold all the server part and a user for it
+
+  ```sql
+  CREATE DATABASE dataservices_db ENCODING = 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8';
+  CREATE USER dataservices_user;
+  ```
+
+- Install needed extensions in `dataservices_db` database
 
     ```
-    create extension cdb_geocoder;
-    create extension plproxy;
-    create extension observatory;
-    create extension cdb_dataservices_server;
-    create extension cdb_dataservices_client;
+    psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS plproxy; COMMIT" -e
+    psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_dataservices_server; COMMIT" -e
     ```
 
+- [optional] install internal geocoder
+   - Make the extension available in postgres
+     ```
+     git clone https://github.com/CartoDB/data-services.git
+     cd data-services/geocoder/extension
+     sudo make install
+     ```
+
+   - Make sure you have `wget` installed because is needed for the next step.
+
+   - Go to `geocoder` folder and execute the `geocoder_dowload_dumps` script to download the internal geocoder data.
+
+   - Once the data is downloaded, execute this command:
+     ```bash
+     geocoder_restore_dump postgres dataservices_db {DOWNLOADED_DUMPS_FOLDER}/*.sql
+     ```
+
+   - Now we have to make available the extension to be installed by postgres. Follow [this](https://github.com/CartoDB/data-services/tree/master/geocoder/extension) instructions.
+
+   - Now install the extension with:
+     ```
+     psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_geocoder; COMMIT" -e
+     ```
+
+- [optional] install data observatory extension
+   - Make the extension available in postgresql to be installed
+     ```
+     git clone https://github.com/CartoDB/observatory-extension.git
+     cd observatory
+     sudo make install
+     ```
+   - This extension needs data, dumps are not available so we're going to use the test fixtures to make it work. Execute:
+     ```
+     psql -U postgres -d dataservices_db -f src/pg/test/fixtures/load_fixtures.sql
+     ```
+   - Give permission to execute and select to the `dataservices_user` user:
+     ```
+     psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatoru; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO dataservices_user; COMMIT" -e
+     ```
 
 ### Server configuration
 

client/cdb_dataservices_client--0.16.0--0.17.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.17.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_type text;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_extra jsonb;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_tags jsonb;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_type text;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_extra jsonb;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_tags jsonb;

client/cdb_dataservices_client--0.17.0--0.16.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.16.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_type;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_extra;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_tags;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_type;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_extra;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_tags;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.15.0'
+default_version = '0.17.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.15.0--0.16.0.sql

@@ -0,0 +1,277 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.16.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit (service text)
+RETURNS json AS $$
+DECLARE
+  ret json;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (service text)
+RETURNS json AS $$
+DECLARE
+  ret json;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    SELECT cdb_dataservices_client._cdb_service_get_rate_limit(username, orgname, service) INTO ret; RETURN ret;
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+         RETURN ret;
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    PERFORM cdb_dataservices_client._cdb_service_set_user_rate_limit(username, orgname, service, rate_limit);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    PERFORM cdb_dataservices_client._cdb_service_set_org_rate_limit(username, orgname, service, rate_limit);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json)
+RETURNS void AS $$
+DECLARE
+
+
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+    -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    PERFORM cdb_dataservices_client._cdb_service_set_server_rate_limit(username, orgname, service, rate_limit);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text)
+RETURNS json AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT cdb_dataservices_server.cdb_service_get_rate_limit (username, orgname, service);
+
+$$ LANGUAGE plproxy;
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json)
+RETURNS void AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT cdb_dataservices_server.cdb_service_set_user_rate_limit (username, orgname, service, rate_limit);
+
+$$ LANGUAGE plproxy;
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json)
+RETURNS void AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT cdb_dataservices_server.cdb_service_set_org_rate_limit (username, orgname, service, rate_limit);
+
+$$ LANGUAGE plproxy;
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json)
+RETURNS void AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+
+  SELECT cdb_dataservices_server.cdb_service_set_server_rate_limit (username, orgname, service, rate_limit);
+
+$$ LANGUAGE plproxy;
+
+
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_set_user_rate_limit (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_set_org_rate_limit (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_set_server_rate_limit (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (username text ,orgname text ,service text ,rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit (username text, orgname text, service text) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_user_rate_limit (username text, orgname text, service text, rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_org_rate_limit (username text, orgname text, service text, rate_limit json) FROM PUBLIC, publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_set_server_rate_limit (username text, orgname text, service text, rate_limit json) FROM PUBLIC, publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_get_rate_limit(service text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe(service text )  TO publicuser;

client/old_versions/cdb_dataservices_client--0.16.0--0.15.0.sql

@@ -0,0 +1,21 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.15.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_get_rate_limit (text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_set_user_rate_limit (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_set_org_rate_limit (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_set_server_rate_limit (text, text, text, json);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit_exception_safe (text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit_exception_safe (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit_exception_safe (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit_exception_safe (text, text, text, json);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_get_rate_limit (text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_user_rate_limit (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_org_rate_limit (text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_set_server_rate_limit (text, text, text, json);

client/renderer/interface.yaml

@@ -421,3 +421,29 @@
   params:
     - { name: service, type: TEXT }
     - { name: input_size, type: NUMERIC }
+
+- name: cdb_service_get_rate_limit
+  return_type: json
+  params:
+    - { name: service,  type: "text" }
+
+- name: cdb_service_set_user_rate_limit
+  superuser: true
+  return_type: void
+  params:
+    - { name: service,  type: "text" }
+    - { name: rate_limit, type: json }
+
+- name: cdb_service_set_org_rate_limit
+  superuser: true
+  return_type: void
+  params:
+    - { name: service,  type: "text" }
+    - { name: rate_limit, type: json }
+
+- name: cdb_service_set_server_rate_limit
+  superuser: true
+  return_type: void
+  params:
+    - { name: service,  type: "text" }
+    - { name: rate_limit, type: json }

client/renderer/sql-template-renderer

@@ -17,7 +17,7 @@ class SqlTemplateRenderer
   end
 
   def render
-    ERB.new(@template).result(binding)
+    ERB.new(@template, _save_level=nil, _trim_mode='-').result(binding)
   end
 
   def name
@@ -44,16 +44,29 @@ class SqlTemplateRenderer
     @function_signature['geocoder_config_key']
   end
 
-  def params
-    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}
+  def parameters_info(with_user_org)
+    parameters = []
+    if with_user_org
+      parameters << { 'name' => 'username', 'type' => 'text' }
+      parameters << { 'name' => 'orgname',  'type' => 'text' }
+    end
+    parameters + @function_signature['params'].reject(&:empty?)
   end
 
-  def params_with_type
-    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }
+  def user_org_declaration()
+    "username text;\n  orgname text;" unless superuser_function?
   end
 
-  def params_with_type_and_default
-    parameters = @function_signature['params'].reject(&:empty?).map do |p|
+  def params(with_user_org = superuser_function?)
+    parameters_info(with_user_org).map { |p| p['name'].to_s }
+  end
+
+  def params_with_type(with_user_org = superuser_function?)
+    parameters_info(with_user_org).map { |p| "#{p['name']} #{p['type']}" }
+  end
+
+  def params_with_type_and_default(with_user_org = superuser_function?)
+    parameters = parameters_info(with_user_org).map do |p|
       if not p['default'].nil?
         "#{p['name']} #{p['type']} DEFAULT #{p['default']}"
       else
@@ -62,6 +75,49 @@ class SqlTemplateRenderer
     end
     return parameters
   end
+
+  def superuser_function?
+    !!@function_signature['superuser']
+  end
+
+  def void_return_type?
+    return_type.downcase == 'void'
+  end
+
+  def return_declaration
+     "ret #{return_type};" unless void_return_type? || multi_row
+  end
+
+  def return_statement(&block)
+    if block
+      erb_out = block.binding.eval('_erbout')
+
+      if multi_row
+        erb_out << 'RETURN QUERY SELECT * FROM '
+      elsif multi_field
+        erb_out << 'SELECT * FROM '
+      elsif void_return_type?
+        erb_out << 'PERFORM '
+      else
+        erb_out << 'SELECT '
+      end
+      yield
+      if multi_row || void_return_type?
+        erb_out << ';'
+      else
+        erb_out << ' INTO ret;'
+      end
+      if !multi_row && !void_return_type?
+        erb_out << ' RETURN ret;'
+      end
+    else
+      if !multi_row && !void_return_type?
+        ' RETURN ret;'
+      end
+    end
+  end
+
+
 end
 
 

client/renderer/templates/20_public_functions.erb

@@ -7,27 +7,18 @@
 CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default.join(' ,') %>)
 RETURNS <%= return_type %> AS $$
 DECLARE
-  <% if not multi_row %>ret <%= return_type %>;<% end %>
-  username text;
-  orgname text;
+  <%= return_declaration if not multi_row %>
+  <%= user_org_declaration %>
 BEGIN
   IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
     RAISE EXCEPTION 'The api_key must be provided';
   END IF;
-  SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
+  <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %>
   -- JSON value stored "" is taken as literal
   IF username IS NULL OR username = '' OR username = '""' THEN
     RAISE EXCEPTION 'Username is a mandatory argument, check it out';
   END IF;
-  <% if multi_row %>
-    RETURN QUERY
-    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
-  <% elsif multi_field %>
-    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
-    RETURN ret;
-  <% else %>
-    SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
-    RETURN ret;
-  <% end %>
+
+  <% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %>
 END;
 $$ LANGUAGE 'plpgsql' SECURITY DEFINER;

client/renderer/templates/25_exception_safe_private_functions.erb

@@ -5,9 +5,8 @@
 CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe (<%= params_with_type_and_default.join(' ,') %>)
 RETURNS <%= return_type %> AS $$
 DECLARE
-  <% if not multi_row %>ret <%= return_type %>;<% end %>
-  username text;
-  orgname text;
+  <%= return_declaration %>
+  <%= user_org_declaration %>
   _returned_sqlstate TEXT;
   _message_text TEXT;
   _pg_exception_context TEXT;
@@ -15,47 +14,22 @@ BEGIN
   IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
     RAISE EXCEPTION 'The api_key must be provided';
   END IF;
-  SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
+  <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %>
   -- JSON value stored "" is taken as literal
   IF username IS NULL OR username = '' OR username = '""' THEN
     RAISE EXCEPTION 'Username is a mandatory argument, check it out';
   END IF;
 
-  <% if multi_row %>
-    BEGIN
-      RETURN QUERY
-      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>);
-    EXCEPTION
-      WHEN OTHERS THEN
+
+  BEGIN
+    <% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %>
+  EXCEPTION
+    WHEN OTHERS THEN
         GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
                                 _message_text = MESSAGE_TEXT,
                                 _pg_exception_context = PG_EXCEPTION_CONTEXT;
         RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
-    END;
-  <% elsif multi_field %>
-    BEGIN
-      SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
-      RETURN ret;
-    EXCEPTION
-      WHEN OTHERS THEN
-        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
-                                _message_text = MESSAGE_TEXT,
-                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
-        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
-        RETURN ret;
-    END;
-  <% else %>
-    BEGIN
-      SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= ['username', 'orgname'].concat(params).join(', ') %>) INTO ret;
-      RETURN ret;
-    EXCEPTION
-      WHEN OTHERS THEN
-        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
-                                _message_text = MESSAGE_TEXT,
-                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
-        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
-        RETURN ret;
-    END;
-  <% end %>
+        <%= return_statement %>
+  END;
 END;
 $$ LANGUAGE 'plpgsql' SECURITY DEFINER;

client/renderer/templates/30_plproxy_functions.erb

@@ -1,9 +1,9 @@
-CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= ['username text', 'organization_name text'].concat(params_with_type_and_default).join(', ') %>)
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (<%= params_with_type_and_default(_with_user_org=true).join(', ') %>)
 RETURNS <%= return_type %> AS $$
   CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str();
   <% if multi_field %>
-  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
+  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>);
   <% else %>
-  SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= ['username', 'organization_name'].concat(params).join(', ') %>);
+  SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (<%= params(_with_user_org=true).join(', ') %>);
   <% end %>
 $$ LANGUAGE plproxy;

client/renderer/templates/90_grant_execute.erb

@@ -1,2 +1,4 @@
+<% unless superuser_function? %>
 GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %>(<%= params_with_type.join(', ') %>) TO publicuser;
-GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %>) TO publicuser;
+GRANT EXECUTE ON FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>_exception_safe(<%= params_with_type.join(', ') %> )  TO publicuser;
+<% end %>

client/sql/16_custom_types.sql

@@ -15,9 +15,9 @@ CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_
 
 CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean);
 
-CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric);
+CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
 
-CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean);
+CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
 
 
 -- For quotas and services configuration

doc/rate_limits.md

@@ -0,0 +1,197 @@
+# Rate limits
+
+Services can be rate-limited. (currently only gecoding is limited)
+
+The rate limits configuration can be established at server, organization or user levels, the latter having precedence over the earlier.
+
+The default configuration (a null or empty configuration) doesn't impose any limits.
+
+The configuration consist of a JSON object with two attributes:
+
+* `period`: the rate-limiting period, in seconds.
+* `limit`: the maximum number of request in the established period.
+
+If a service request exceeds the configured rate limits
+(i.e. if more than `limit` calls are performe in a fixed interval of
+duration `period` seconds) the call will fail with an "Rate limit exceeded" error.
+
+## Server-side interface
+
+There's a server-side SQL interface to query or change the configuration.
+
+### cdb_dataservices_server.cdb_service_get_rate_limit(username, orgname, service)
+
+This function returns the rate limit configuration for a given user and service.
+
+#### Returns
+
+The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
+
+### cdb_dataservices_server.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the rate limit configuration for the user. This overrides any other configuration.
+
+The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+### cdb_dataservices_server.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the rate limit configuration for the organization.
+This overrides server level configuration and is overriden by user configuration if present.
+
+The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+### cdb_dataservices_server.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
+
+The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+## Client-side interface
+
+For convenience there's also a client-side interface (in the client dataservices-api extension), consisting
+of public functions to get the current configuration and privileged functions to change it.
+
+### Public functions
+
+These functions are accesible to non-privileged roles, and should only be executed
+using the role corresponding to a CARTO user, since that will determine the
+user and organization to which the rate limits configuration applies.
+
+### cdb_dataservices_client.cdb_service_get_rate_limit(service)
+
+This function returns the rate limit configuration in effect for the specified service
+and the user corresponding to the role which makes the calls. The effective configuration
+may come from any of the configuration levels (server/organization/user); only the
+existing configuration with most precedence is returned.
+
+#### Returns
+
+The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
+
+#### Example:
+
+```
+SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoder');
+
+   cdb_service_get_rate_limit
+---------------------------------
+ {"limit": 1000, "period": 86400}
+(1 row)
+```
+
+
+### Privileged (superuser) functions
+
+Thes functions are not accessible by regular user roles, and the user and organization names must be provided as parameters.
+
+### cdb_dataservices_client.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the rate limit configuration for the user. This overrides any other configuration.
+
+The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+#### Example
+
+This will configure the geocoder service rate limit for  user  `myusername`, a non-organization user.
+The limit will be set at 1000 requests per day. Since the user doesn't belong to any organization,
+`NULL` will be passed to the organization argument; otherwise the name of the user's organization should
+be provided.
+
+Note that the name of the geocoding services is `geocoder` and not `geocoding`.
+
+```
+SELECT cdb_dataservices_client.cdb_service_set_user_rate_limit(
+    'myusername',
+    NULL,
+    'geocoder',
+    '{"limit":1000,"period":86400}'
+);
+
+ cdb_service_set_user_rate_limit
+---------------------------------
+
+(1 row)
+```
+
+### cdb_dataservices_client.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the rate limit configuration for the organization.
+This overrides server level configuration and is overriden by user configuration if present.
+
+The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+#### Example
+
+This will configure the geocoder service rate limit for the `myorg` organization.
+The limit will be set at 100 requests per hour.
+Note that even we're setting the default configuration for the whole organization,
+the name of a user of the organization must be provided for technical reasons.
+
+```
+SELECT cdb_dataservices_client.cdb_service_set_org_rate_limit(
+    'myorgadmin',
+    'myorg',
+    'geocoder',
+    '{"limit":100,"period":3600}'
+);
+
+
+ cdb_service_set_org_rate_limit
+---------------------------------
+
+(1 row)
+```
+
+### cdb_dataservices_client.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
+
+This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
+
+The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
+
+#### Returns
+
+This functions doesn't return any value.
+
+#### Example
+
+This will configure the default geocoder service rate limit for all users
+accesing the data-services server.
+The limit will be set at 10000 requests per month.
+Note that even we're setting the default configuration for the server,
+the name of a user and the name of the corresponding organization (or NULL)
+must be provided for technical reasons.
+
+```
+SELECT cdb_dataservices_client.cdb_service_set_server_rate_limit(
+    'myorgadmin',
+    'myorg',
+    'geocoder',
+    '{"limit":10000,"period":108000}'
+);
+
+
+ cdb_service_set_server_rate_limit
+---------------------------------
+
+(1 row)
+```

server/extension/cdb_dataservices_server--0.23.0--0.24.0.sql

@@ -0,0 +1,11 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.24.0'" to load this file. \quit
+
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_type text;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_extra jsonb;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_tags jsonb;
+
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_type text;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_extra jsonb;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_tags jsonb;

server/extension/cdb_dataservices_server--0.24.0--0.23.0.sql

@@ -0,0 +1,12 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.23.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_type;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_extra;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_tags;
+
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_type;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_extra;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_tags;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.22.0'
+default_version = '0.24.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.22.0--0.23.0.sql

@@ -0,0 +1,207 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.23.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT)
+RETURNS JSON AS $$
+  import json
+  from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_config = ServiceConfiguration(service, username, orgname)
+  rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
+  if rate_limit_config.is_limited():
+      return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
+  else:
+      return None
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_user_rate_limits(config)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_org_rate_limits(config)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_server_rate_limits(config)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE
+FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.here import HereMapsGeocoder
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  service_manager.assert_within_limits()
+
+  try:
+    geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using here maps')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.tools import LegacyServiceManager
+  from cartodb_services.google import GoogleMapsGeocoder
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  service_manager.assert_within_limits(quota=False)
+
+  try:
+    geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using google maps')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.mapzen import MapzenGeocoder
+  from cartodb_services.mapzen.types import country_to_iso3
+  from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname)
+  service_manager.assert_within_limits()
+
+  try:
+    geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params)
+    country_iso3 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3, search_type='address')
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using mapzen')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu;

server/extension/old_versions/cdb_dataservices_server--0.23.0--0.22.0.sql

@@ -0,0 +1,15 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.22.0'" to load this file. \quit
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_get_rate_limit(TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_set_user_rate_limit(TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_set_org_rate_limit(TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_service_set_server_rate_limit(TEXT, TEXT, TEXT, JSON);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.CDB_Conf_SetConf(text, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.CDB_Conf_RemoveConf(text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT,  TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT,  TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_geocode_street_point(TEXT, TEXT, TEXT, TEXT,  TEXT, TEXT);

server/extension/sql/130_data_observatory_helper.sql

@@ -35,7 +35,7 @@ RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$
   SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan);
 $$ LANGUAGE plproxy;
 
-CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric);
+CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries(
   username TEXT,
@@ -50,7 +50,7 @@ RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$
   SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan);
 $$ LANGUAGE plproxy;
 
-CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean);
+CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans(
   username TEXT,

server/extension/sql/15_config_helper.sql

@@ -16,6 +16,24 @@ RETURNS JSON AS $$
     SELECT VALUE FROM cartodb.cdb_conf WHERE key = input_key;
 $$ LANGUAGE SQL STABLE SECURITY DEFINER;
 
+CREATE OR REPLACE
+FUNCTION cdb_dataservices_server.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cdb_dataservices_server.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cdb_dataservices_server.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE SECURITY DEFINER;
+
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text, provider text DEFAULT NULL)
 RETURNS boolean AS $$
   cache_key = "user_geocoder_config_{0}".format(username)

server/extension/sql/20_geocode_street.sql

@@ -72,109 +72,77 @@ $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
+  from cartodb_services.tools import LegacyServiceManager
   from cartodb_services.here import HereMapsGeocoder
-  from cartodb_services.metrics import QuotaService
-  from cartodb_services.tools import Logger,LoggerConfig
-
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
-  logger_config = GD["logger_config"]
-  logger = Logger(logger_config)
-  # -- Check the quota
-  quota_service = QuotaService(user_geocoder_config, redis_conn)
-  if not quota_service.check_user_quota():
-    raise Exception('You have reached the limit of your quota')
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  service_manager.assert_within_limits()
 
   try:
-    geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params)
+    geocoder = HereMapsGeocoder(service_manager.config.heremaps_app_id, service_manager.config.heremaps_app_code, service_manager.logger, service_manager.config.heremaps_service_params)
     coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
     if coordinates:
-      quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_success_service_use()
       plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
       point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
       return point['st_setsrid']
     else:
-      quota_service.increment_empty_service_use()
+      service_manager.quota_service.increment_empty_service_use()
       return None
   except BaseException as e:
     import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using here maps', sys.exc_info(), data={"username": username, "orgname": orgname})
     raise Exception('Error trying to geocode street point using here maps')
   finally:
-    quota_service.increment_total_service_use()
+    service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
+  from cartodb_services.tools import LegacyServiceManager
   from cartodb_services.google import GoogleMapsGeocoder
-  from cartodb_services.metrics import QuotaService
-  from cartodb_services.tools import Logger,LoggerConfig
-
-  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
-  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
-  logger_config = GD["logger_config"]
-  logger = Logger(logger_config)
-  quota_service = QuotaService(user_geocoder_config, redis_conn)
+  service_manager = LegacyServiceManager('geocoder', username, orgname, GD)
+  service_manager.assert_within_limits(quota=False)
 
   try:
-    geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger)
+    geocoder = GoogleMapsGeocoder(service_manager.config.google_client_id, service_manager.config.google_api_key, service_manager.logger)
     coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
     if coordinates:
-      quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_success_service_use()
       plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
       point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
       return point['st_setsrid']
     else:
-      quota_service.increment_empty_service_use()
+      service_manager.quota_service.increment_empty_service_use()
       return None
   except BaseException as e:
     import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using google maps', sys.exc_info(), data={"username": username, "orgname": orgname})
     raise Exception('Error trying to geocode street point using google maps')
   finally:
-    quota_service.increment_total_service_use()
+    service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
-  import cartodb_services
-  cartodb_services.init(plpy, GD)
+  from cartodb_services.tools import ServiceManager
   from cartodb_services.mapzen import MapzenGeocoder
   from cartodb_services.mapzen.types import country_to_iso3
-  from cartodb_services.metrics import QuotaService
-  from cartodb_services.tools import Logger
-  from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
   from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
-  from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
-  from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
-  from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
-  from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
-  from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
 
-  server_config_backend = ServerConfigBackendFactory().get()
-  environment = ServerEnvironmentBuilder(server_config_backend).get()
-  user_config_backend = UserConfigBackendFactory(username, environment, server_config_backend).get()
-  org_config_backend = OrgConfigBackendFactory(orgname, environment, server_config_backend).get()
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
 
-  logger_config = LoggerConfigBuilder(environment, server_config_backend).get()
-  logger = Logger(logger_config)
-
-  mapzen_geocoder_config = MapzenGeocoderConfigBuilder(server_config_backend, user_config_backend, org_config_backend, username, orgname).get()
-
-  redis_metrics_connection = RedisMetricsConnectionFactory(environment, server_config_backend).get()
-
-  quota_service = QuotaService(mapzen_geocoder_config, redis_metrics_connection)
-  if not quota_service.check_user_quota():
-    raise Exception('You have reached the limit of your quota')
+  service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, username, orgname)
+  service_manager.assert_within_limits()
 
   try:
-    geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params)
+    geocoder = MapzenGeocoder(service_manager.config.mapzen_api_key, service_manager.logger, service_manager.config.service_params)
     country_iso3 = None
     if country:
       country_iso3 = country_to_iso3(country)
@@ -182,18 +150,18 @@ RETURNS Geometry AS $$
                                    state_province=state_province,
                                    country=country_iso3, search_type='address')
     if coordinates:
-      quota_service.increment_success_service_use()
+      service_manager.quota_service.increment_success_service_use()
       plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
       point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
       return point['st_setsrid']
     else:
-      quota_service.increment_empty_service_use()
+      service_manager.quota_service.increment_empty_service_use()
       return None
   except BaseException as e:
     import sys
-    quota_service.increment_failed_service_use()
-    logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
     raise Exception('Error trying to geocode street point using mapzen')
   finally:
-    quota_service.increment_total_service_use()
+    service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu;

server/extension/sql/210_rates.sql

@@ -0,0 +1,91 @@
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_get_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT)
+RETURNS JSON AS $$
+  import json
+  from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_config = ServiceConfiguration(service, username, orgname)
+  rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
+  if rate_limit_config.is_limited():
+      return json.dumps({'limit': rate_limit_config.limit, 'period': rate_limit_config.period})
+  else:
+      return None
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_user_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_user_rate_limits(config)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_org_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_org_rate_limits(config)
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_service_set_server_rate_limit(
+  username TEXT,
+  orgname TEXT,
+  service TEXT,
+  rate_limit_json JSON)
+RETURNS VOID AS $$
+  import json
+  from cartodb_services.config import RateLimitsConfig, RateLimitsConfigSetter
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  config_setter = RateLimitsConfigSetter(service=service, username=username, orgname=orgname)
+  if rate_limit_json:
+      rate_limit = json.loads(rate_limit_json)
+      limit = rate_limit.get('limit', None)
+      period = rate_limit.get('period', None)
+  else:
+      limit = None
+      period = None
+  config = RateLimitsConfig(service=service, username=username, limit=limit, period=period)
+  config_setter.set_server_rate_limits(config)
+$$ LANGUAGE plpythonu;

server/extension/test/expected/210_rates_test.out

@@ -0,0 +1,44 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_get_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_user_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_org_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_server_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+ exists 
+--------
+ t
+(1 row)
+

server/extension/test/sql/210_rates_test.sql

@@ -0,0 +1,27 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_get_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_user_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_org_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = 'cdb_service_set_server_rate_limit'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');

server/lib/python/cartodb_services/README.md

@@ -7,12 +7,7 @@ It is used from pl/python functions contained in the `cdb_dataservices_server` e
 On the other hand, it is pretty independent from the client, as long as the signatures of the public pl/python functions match.
 
 ## Dependencies
-See the [[`requirements.txt`]] or better the Basically:
-- pip
-- redis and hiredis
-- dateutil
-- googlemaps
-- request
+Take a look at [`requirements.txt`](requirements.txt) for details about the required dependencies.
 
 ## Installation
 Install the requirements:
@@ -40,7 +35,7 @@ OK
 ```
 
 ## Running the integration tests
-See the [[../../../../test/README.md]]. Basically, move to the `/test` directory at the top level of this repo and execute the `run_tests.py` script:
+See this [`README`](../../../../test/README.md) in the `/test` directory for details. Basically, you have to move to the `/test` directory at the top level of this repo and execute the `run_tests.py` script:
 ```sh
 cd $(git rev-parse --show-toplevel)/test
 python run_tests.py --host=$YOUR_HOST $YOUR_USERNAME $YOUR_API_KEY

server/lib/python/cartodb_services/cartodb_services/config/__init__.py

@@ -0,0 +1,3 @@
+from service_configuration import ServiceConfiguration
+from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+from legacy_rate_limits import RateLimitsConfigLegacyBuilder

server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py

@@ -0,0 +1,46 @@
+import json
+from rate_limits import RateLimitsConfig
+
+class RateLimitsConfigLegacyBuilder(object):
+    """
+    Build a RateLimitsConfig object using the *legacy* configuration classes
+    """
+
+    def __init__(self, redis_connection, db_conn, service, username, orgname):
+        self._service = service
+        self._username = username
+        self._orgname = orgname
+        self._redis_connection = redis_connection
+        self._db_conn = db_conn
+
+    def get(self):
+        rate_limit = self.__get_rate_limit()
+        return RateLimitsConfig(self._service,
+                                self._username,
+                                rate_limit.get('limit', None),
+                                rate_limit.get('period', None))
+
+    def __get_rate_limit(self):
+        rate_limit = {}
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        user_key = "rails:users:{0}".format(self._username)
+        rate_limit_json = self.__get_redis_config(user_key, rate_limit_key)
+        if not rate_limit_json and self._orgname:
+            org_key = "rails:orgs:{0}".format(self._orgname)
+            rate_limit_json = self.__get_redis_config(org_key, rate_limit_key)
+        if rate_limit_json:
+            rate_limit = json.loads(rate_limit_json)
+        else:
+            conf_key = 'rate_limits'
+            sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(conf_key)
+            try:
+                conf = self._db_conn.execute(sql, 1)[0]['conf']
+            except Exception:
+                conf = None
+            if conf:
+                rate_limit = json.loads(conf).get(self._service)
+        return rate_limit or {}
+
+    def __get_redis_config(self, basekey, param):
+        config = self._redis_connection.hgetall(basekey)
+        return config and config.get(param)

server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py

@@ -0,0 +1,113 @@
+import json
+
+from service_configuration import ServiceConfiguration
+
+class RateLimitsConfig(object):
+    """
+    Value object that represents the configuration needed to rate-limit services
+    """
+
+    def __init__(self,
+                 service,
+                 username,
+                 limit,
+                 period):
+        self._service = service
+        self._username = username
+        self._limit = limit and int(limit)
+        self._period = period and int(period)
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+    # service this limit applies to
+    @property
+    def service(self):
+        return self._service
+
+    # user this limit applies to
+    @property
+    def username(self):
+        return self._username
+
+    # rate period in seconds
+    @property
+    def period(self):
+        return self._period
+
+    # rate limit in seconds
+    @property
+    def limit(self):
+        return self._limit
+
+    def is_limited(self):
+        return self._limit and self._limit > 0 and self._period and self._period > 0
+
+class RateLimitsConfigBuilder(object):
+    """
+    Build a rate limits configuration obtaining the parameters
+    from the user/org/server configuration.
+    """
+
+    def __init__(self, server_conf, user_conf, org_conf, service, username, orgname):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._service = service
+        self._username = username
+        self._orgname = orgname
+
+    def get(self):
+        # Order of precedence is user_conf, org_conf, server_conf
+
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+
+        rate_limit_json = self._user_conf.get(rate_limit_key, None) or self._org_conf.get(rate_limit_key, None)
+        if (rate_limit_json):
+            rate_limit = rate_limit_json and json.loads(rate_limit_json)
+        else:
+            rate_limit = self._server_conf.get('rate_limits', {}).get(self._service, {})
+
+        return RateLimitsConfig(self._service,
+                                self._username,
+                                rate_limit.get('limit', None),
+                                rate_limit.get('period', None))
+
+
+class RateLimitsConfigSetter(object):
+
+    def __init__(self, service, username, orgname=None):
+        self._service = service
+        self._service_config = ServiceConfiguration(service, username, orgname)
+
+    def set_user_rate_limits(self, rate_limits_config):
+        # Note we allow copying a config from another user/service, so we
+        # ignore rate_limits:config.service and rate_limits:config.username
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        if rate_limits_config.is_limited():
+            rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+            rate_limit_json = json.dumps(rate_limit)
+            self._service_config.user.set(rate_limit_key, rate_limit_json)
+        else:
+            self._service_config.user.remove(rate_limit_key)
+
+    def set_org_rate_limits(self, rate_limits_config):
+        rate_limit_key = "{0}_rate_limit".format(self._service)
+        if rate_limits_config.is_limited():
+            rate_limit = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+            rate_limit_json = json.dumps(rate_limit)
+            self._service_config.org.set(rate_limit_key, rate_limit_json)
+        else:
+            self._service_config.org.remove(rate_limit_key)
+
+    def set_server_rate_limits(self, rate_limits_config):
+        rate_limits = self._service_config.server.get('rate_limits', {})
+        if rate_limits_config.is_limited():
+            rate_limits[self._service] = {'limit': rate_limits_config.limit, 'period': rate_limits_config.period}
+        else:
+            rate_limits.pop(self._service, None)
+        if rate_limits:
+            self._service_config.server.set('rate_limits', rate_limits)
+        else:
+            self._service_config.server.remove('rate_limits')
+

server/lib/python/cartodb_services/cartodb_services/config/service_configuration.py

@@ -0,0 +1,36 @@
+from cartodb_services.refactor.core.environment import ServerEnvironmentBuilder
+from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+from cartodb_services.refactor.backend.user_config import UserConfigBackendFactory
+from cartodb_services.refactor.backend.org_config import OrgConfigBackendFactory
+
+class ServiceConfiguration(object):
+    """
+    This class instantiates configuration backend objects for all the configuration levels of a service:
+    * environment
+    * server
+    * organization
+    * user
+    The configuration backends allow retrieval and modification of configuration parameters.
+    """
+
+    def __init__(self, service, username, orgname):
+        self._server_config_backend = ServerConfigBackendFactory().get()
+        self._environment = ServerEnvironmentBuilder(self._server_config_backend ).get()
+        self._user_config_backend = UserConfigBackendFactory(username, self._environment, self._server_config_backend ).get()
+        self._org_config_backend = OrgConfigBackendFactory(orgname, self._environment, self._server_config_backend ).get()
+
+    @property
+    def environment(self):
+        return self._environment
+
+    @property
+    def server(self):
+        return self._server_config_backend
+
+    @property
+    def user(self):
+        return self._user_config_backend
+
+    @property
+    def org(self):
+        return self._org_config_backend

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -551,7 +551,7 @@ class ServicesDBConfig:
 
     def _get_conf(self, key):
         try:
-            sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
+            sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
             conf = self._db_conn.execute(sql, 1)
             return conf[0]['conf']
         except Exception as e:

server/lib/python/cartodb_services/cartodb_services/metrics/user.py

@@ -118,12 +118,9 @@ class UserMetricsService:
         for date in self.__generate_date_range(date_from, date_to):
             redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
                                                      service, metric, date)
-            score = self._redis_connection.zscore(redis_prefix, date.day)
-            aggregated_metric += int(score) if score else 0
             zero_padded_day = date.strftime(self.DAY_OF_MONTH_ZERO_PADDED)
-            if str(date.day) != zero_padded_day:
-                score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
-                aggregated_metric += int(score) if score else 0
+            score = self._redis_connection.zscore(redis_prefix, zero_padded_day)
+            aggregated_metric += int(score) if score else 0
 
         return aggregated_metric
 

server/lib/python/cartodb_services/cartodb_services/refactor/core/interfaces.py

@@ -6,6 +6,6 @@ class ConfigBackendInterface(object):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def get(self, key):
+    def get(self, key, default=None):
         """Return a value based on the key supplied from some storage"""
         pass

server/lib/python/cartodb_services/cartodb_services/refactor/storage/mem_config.py

@@ -5,8 +5,8 @@ class InMemoryConfigStorage(ConfigBackendInterface):
     def __init__(self, config_hash={}):
         self._config_hash = config_hash
 
-    def get(self, key):
+    def get(self, key, default=None):
         try:
             return self._config_hash[key]
         except KeyError:
-            return None
+            return default

server/lib/python/cartodb_services/cartodb_services/refactor/storage/null_config.py

@@ -2,5 +2,5 @@ from ..core.interfaces import ConfigBackendInterface
 
 class NullConfigStorage(ConfigBackendInterface):
 
-    def get(self, key):
-        return None
+    def get(self, key, default=None):
+        return default

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -9,11 +9,19 @@ class RedisConfigStorage(ConfigBackendInterface):
         self._config_key = config_key
         self._data = None
 
-    def get(self, key):
+    def get(self, key, default=KeyError):
         if not self._data:
             self._data = self._connection.hgetall(self._config_key)
-        return self._data[key]
+        if (default == KeyError):
+            return self._data[key]
+        else:
+            return self._data.get(key, default)
 
+    def set(self, key, value):
+        self._connection.hset(self._config_key, key, value)
+
+    def remove(self, key):
+        self._connection.hdel(self._config_key, key)
 
 class RedisUserConfigStorageBuilder(object):
     def __init__(self, redis_connection, username):

server/lib/python/cartodb_services/cartodb_services/refactor/storage/server_config.py

@@ -4,11 +4,28 @@ from ..core.interfaces import ConfigBackendInterface
 
 class InDbServerConfigStorage(ConfigBackendInterface):
 
-    def get(self, key):
+    def get(self, key, default=None):
         sql = "SELECT cdb_dataservices_server.cdb_conf_getconf('{0}') as conf".format(key)
         rows = cartodb_services.plpy.execute(sql, 1)
-        json_output = rows[0]['conf']
-        if json_output:
+        json_output = None
+        try:
+            json_output = rows[0]['conf']
+        except (IndexError, KeyError):
+            pass
+        if (json_output):
             return json.loads(json_output)
         else:
-            return None
+            if (default == KeyError):
+                raise KeyError
+            else:
+                return default
+
+    def set(self, key, config):
+        json_config = json.dumps(config)
+        quoted_config = cartodb_services.plpy.quote_nullable(json_config)
+        sql = "SELECT cdb_dataservices_server.cdb_conf_setconf('{0}', {1})".format(key, quoted_config)
+        cartodb_services.plpy.execute(sql)
+
+    def remove(self, key):
+        sql = "SELECT cdb_dataservices_server.cdb_conf_removeconf('{0}')".format(key)
+        cartodb_services.plpy.execute(sql)

server/lib/python/cartodb_services/cartodb_services/tools/__init__.py

@@ -2,3 +2,6 @@ from redis_tools import RedisConnection, RedisDBConfig
 from coordinates import Coordinate
 from polyline import PolyLine
 from log import Logger, LoggerConfig
+from rate_limiter import RateLimiter
+from service_manager import ServiceManager, RateLimitExceeded
+from legacy_service_manager import  LegacyServiceManager

server/lib/python/cartodb_services/cartodb_services/tools/legacy_service_manager.py

@@ -0,0 +1,23 @@
+from cartodb_services.metrics import QuotaService
+from cartodb_services.tools import Logger,LoggerConfig
+from cartodb_services.tools import RateLimiter
+from cartodb_services.config import RateLimitsConfigLegacyBuilder
+from cartodb_services.tools.service_manager import ServiceManagerBase
+import plpy
+
+class LegacyServiceManager(ServiceManagerBase):
+    """
+    This service manager relies on cached configuration (in gd) stored in *legacy* configuration objects
+    It's intended for use by the *legacy* configuration objects (in use prior to the configuration refactor).
+    """
+
+    def __init__(self, service, username, orgname, gd):
+        redis_conn = gd["redis_connection_{0}".format(username)]['redis_metrics_connection']
+        self.config = gd["user_{0}_config_{1}".format(service, username)]
+        logger_config = gd["logger_config"]
+        self.logger = Logger(logger_config)
+
+        self.quota_service = QuotaService(self.config, redis_conn)
+
+        rate_limit_config = RateLimitsConfigLegacyBuilder(redis_conn, plpy, service=service, username=username, orgname=orgname).get()
+        self.rate_limiter = RateLimiter(rate_limit_config, redis_conn)

server/lib/python/cartodb_services/cartodb_services/tools/log.py

@@ -185,7 +185,7 @@ class LoggerConfig:
 
     def _get_conf(self, key):
         try:
-            sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
+            sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
             conf = self._db_conn.execute(sql, 1)
             return conf[0]['conf']
         except Exception as e:

server/lib/python/cartodb_services/cartodb_services/tools/rate_limiter.py

@@ -0,0 +1,18 @@
+from rratelimit import Limiter
+
+class RateLimiter:
+
+    def __init__(self, rate_limits_config, redis_connection):
+        self._config = rate_limits_config
+        self._limiter = None
+        if (self._config.is_limited()):
+            self._limiter = Limiter(redis_connection,
+                                    action=self._config.service,
+                                    limit=self._config.limit,
+                                    period=self._config.period)
+
+    def check(self):
+        ok = True
+        if (self._limiter):
+            ok = self._limiter.checked_insert(self._config.username)
+        return ok

server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py

@@ -37,7 +37,7 @@ class RedisDBConfig:
         return self._build(key)
 
     def _build(self, key):
-        conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(
+        conf_query = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(
             key)
         conf = self._db_conn.execute(conf_query)[0]['conf']
         if conf is None:

server/lib/python/cartodb_services/cartodb_services/tools/service_manager.py

@@ -0,0 +1,70 @@
+from cartodb_services.metrics import QuotaService
+from cartodb_services.tools import Logger
+from cartodb_services.tools import RateLimiter
+from cartodb_services.refactor.tools.logger import LoggerConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisMetricsConnectionFactory
+from cartodb_services.config import ServiceConfiguration, RateLimitsConfigBuilder
+
+class RateLimitExceeded(Exception):
+    def __str__(self):
+            return repr('Rate limit exceeded')
+
+class ServiceManagerBase:
+    """
+    A Service manager collects the configuration needed to use a service,
+    including thir-party services parameters.
+
+    This abstract class serves as the base for concrete service manager classes;
+    derived class must provide and initialize attributes for ``config``,
+    ``quota_service``, ``logger`` and ``rate_limiter`` (which can be None
+    for no limits).
+
+    It provides an `assert_within_limits` method to check quota and rate limits
+    which raises exceptions when limits are exceeded.
+
+    It exposes properties containing:
+
+    * ``config`` : a configuration object containing the configuration parameters for
+      a given service and provider.
+    * ``quota_service`` a QuotaService object to for quota accounting
+    * ``logger``
+
+    """
+
+    def assert_within_limits(self, quota=True, rate=True):
+        if rate and not self.rate_limiter.check():
+            raise RateLimitExceeded()
+        if quota and not self.quota_service.check_user_quota():
+            raise Exception('You have reached the limit of your quota')
+
+    @property
+    def config(self):
+        return self.config
+
+    @property
+    def quota_service(self):
+        return self.quota_service
+
+    @property
+    def logger(self):
+        return self.logger
+class ServiceManager(ServiceManagerBase):
+    """
+    This service manager delegates the configuration parameter details,
+    and the policies about configuration precedence to a configuration-builder class.
+    It uses the refactored configuration classes.
+    """
+
+    def __init__(self, service, config_builder, username, orgname):
+        service_config = ServiceConfiguration(service, username, orgname)
+
+        logger_config = LoggerConfigBuilder(service_config.environment, service_config.server).get()
+        self.logger = Logger(logger_config)
+
+        self.config = config_builder(service_config.server, service_config.user, service_config.org, username, orgname).get()
+        rate_limit_config = RateLimitsConfigBuilder(service_config.server, service_config.user, service_config.org, service=service, username=username, orgname=orgname).get()
+
+        redis_metrics_connection = RedisMetricsConnectionFactory(service_config.environment, service_config.server).get()
+
+        self.rate_limiter = RateLimiter(rate_limit_config, redis_metrics_connection)
+        self.quota_service = QuotaService(self.config, redis_metrics_connection)

server/lib/python/cartodb_services/requirements.txt

@@ -4,7 +4,8 @@ python-dateutil==2.2
 googlemaps==2.4.2
 rollbar==0.13.2
 # Dependency for googlemaps package
-requests<=2.9.1
+requests==2.9.1
+rratelimit==0.0.4
 
 # Test
 mock==1.3.0

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.14.0',
+    version='0.15.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/mock_plpy.py

@@ -16,7 +16,7 @@ class MockPlPy:
     def __init__(self):
         self._reset()
 
-    def _reset(self):
+    def _reset(self, log_executed_queries=False):
         self.infos = []
         self.notices = []
         self.debugs = []
@@ -28,11 +28,30 @@ class MockPlPy:
         self.results = []
         self.prepares = []
         self.results = {}
+        self._log_executed_queries = log_executed_queries
+        self._logged_queries = []
 
     def _define_result(self, query, result):
         pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
         self.results[pattern] = result
 
+    def _executed_queries(self):
+        if self._log_executed_queries:
+            return self._logged_queries
+        else:
+            raise Exception('Executed queries logging is not active')
+
+    def _has_executed_query(self, query):
+        pattern = re.compile(re.escape(query))
+        for executed_query in self._executed_queries():
+            if pattern.search(executed_query):
+                return True
+        return False
+
+    def _start_logging_executed_queries(self):
+        self._logged_queries = []
+        self._log_executed_queries = True
+
     def notice(self, msg):
         self.notices.append(msg)
 
@@ -47,7 +66,15 @@ class MockPlPy:
         return MockCursor(data)
 
     def execute(self, query, rows=1):
+        if self._log_executed_queries:
+            self._logged_queries.append(query)
         for pattern, result in self.results.iteritems():
             if pattern.search(query):
                 return result
         return []
+
+    def quote_nullable(self, value):
+        if value is None:
+            return 'NULL'
+        else:
+            return "'{0}'".format(value)

server/lib/python/cartodb_services/test/test_ratelimitsconfig.py

@@ -0,0 +1,124 @@
+from test_helper import *
+from unittest import TestCase
+from mock import Mock, MagicMock, patch
+from nose.tools import assert_raises, assert_not_equal, assert_equal
+
+from datetime import datetime, date
+from mockredis import MockRedis
+import cartodb_services
+
+from cartodb_services.tools import ServiceManager, LegacyServiceManager
+
+from cartodb_services.metrics import GeocoderConfig
+from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
+from cartodb_services.tools import RateLimitExceeded
+
+from cartodb_services.refactor.storage.redis_config import *
+from cartodb_services.refactor.storage.mem_config import InMemoryConfigStorage
+from cartodb_services.refactor.backend.server_config import ServerConfigBackendFactory
+from cartodb_services.config import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+
+class TestRateLimitsConfig(TestCase):
+
+    def setUp(self):
+        plpy_mock_config()
+        cartodb_services.init(plpy_mock, _GD={})
+        self.username = 'test_user'
+        self.orgname = 'test_org'
+        self.redis_conn = MockRedis()
+        build_redis_user_config(self.redis_conn, self.username, 'geocoding')
+        build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
+        self.environment = 'production'
+        plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        basic_server_conf = {"server_conf": {"environment": "testing"},
+                                                     "mapzen_conf":
+                                                     {"geocoder":
+                                                      {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}}
+                                                     }, "logger_conf": {}}
+        self.empty_server_config = InMemoryConfigStorage(basic_server_conf)
+        self.empty_redis_config = InMemoryConfigStorage({})
+        self.user_config = RedisUserConfigStorageBuilder(self.redis_conn, self.username).get()
+        self.org_config = RedisOrgConfigStorageBuilder(self.redis_conn, self.orgname).get()
+        self.server_config = ServerConfigBackendFactory().get()
+
+    def test_server_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            # Write server level configuration
+            config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+            plpy_mock._start_logging_executed_queries()
+            config_setter.set_server_rate_limits(config)
+            assert plpy_mock._has_executed_query('cdb_conf_setconf(\'rate_limits\', \'{"geocoder": {"limit": 1234, "period": 86400}}\')')
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.empty_redis_config,
+                org_conf=self.empty_redis_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, config)
+
+    def test_server_org_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+
+            # Write server level configuration
+            config_setter.set_server_rate_limits(server_config)
+            # Override with org level configuration
+            config_setter.set_org_rate_limits(org_config)
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.empty_redis_config,
+                org_conf=self.org_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, org_config)
+
+    def test_server_org_user_config(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+
+            server_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1234, period=86400)
+            org_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1235, period=86400)
+            user_config = RateLimitsConfig(service='geocoder', username=self.username, limit=1236, period=86400)
+            config_setter = RateLimitsConfigSetter(service='geocoder', username=self.username, orgname=self.orgname)
+
+            # Write server level configuration
+            config_setter.set_server_rate_limits(server_config)
+            # Override with org level configuration
+            config_setter.set_org_rate_limits(org_config)
+            # Override with user level configuration
+            config_setter.set_user_rate_limits(user_config)
+
+            # Re-read configuration
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': '{"geocoder": {"limit": 1234, "period": 86400}}'}])
+            read_config = RateLimitsConfigBuilder(
+                server_conf=self.server_config,
+                user_conf=self.user_config,
+                org_conf=self.org_config,
+                service='geocoder',
+                username=self.username,
+                orgname=self.orgname
+            ).get()
+            plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+            assert_equal(read_config, user_config)

server/lib/python/cartodb_services/test/test_servicemanager.py

@@ -0,0 +1,218 @@
+from test_helper import *
+from unittest import TestCase
+from mock import Mock, MagicMock, patch
+from nose.tools import assert_raises, assert_not_equal, assert_equal
+
+from datetime import datetime, date
+from cartodb_services.tools import ServiceManager, LegacyServiceManager
+from mockredis import MockRedis
+import cartodb_services
+from cartodb_services.metrics import GeocoderConfig
+from cartodb_services.refactor.service.mapzen_geocoder_config import MapzenGeocoderConfigBuilder
+from cartodb_services.refactor.backend.redis_metrics_connection import RedisConnectionBuilder
+from cartodb_services.tools import RateLimitExceeded
+
+LUA_AVAILABLE_FOR_MOCKREDIS = False
+
+class MockRedisWithVersionInfo(MockRedis):
+    def info(self):
+        return {'redis_version': '3.0.2'}
+
+class TestServiceManager(TestCase):
+
+    def setUp(self):
+        plpy_mock_config()
+        cartodb_services.init(plpy_mock, _GD={})
+        self.username = 'test_user'
+        self.orgname = 'test_org'
+        self.redis_conn = MockRedisWithVersionInfo()
+        build_redis_user_config(self.redis_conn, self.username, 'geocoding')
+        build_redis_org_config(self.redis_conn, self.orgname, 'geocoding', provider='mapzen')
+        self.environment = 'production'
+        plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "production"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metadata_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+        plpy_mock._define_result("CDB_Conf_GetConf\('redis_metrics_config'\)", [{'conf': '{"redis_host":"localhost","redis_port":"6379"}'}])
+
+    def check_rate_limit(self, service_manager, n, active=True):
+        if LUA_AVAILABLE_FOR_MOCKREDIS:
+            for _ in xrange(n):
+                service_manager.assert_within_limits()
+            if active:
+                with assert_raises(RateLimitExceeded):
+                    service_manager.assert_within_limits()
+            else:
+                service_manager.assert_within_limits()
+        else:
+            # rratelimit doesn't work with MockRedis because it needs Lua support
+            # so, we'll simply perform some sanity check on the configuration of the rate limiter
+            if active:
+                assert_equal(service_manager.rate_limiter._config.is_limited(), True)
+                assert_equal(service_manager.rate_limiter._config.limit, n)
+            else:
+                assert not service_manager.rate_limiter._config.is_limited()
+
+    def test_legacy_service_manager(self):
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        service_manager.assert_within_limits()
+        assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
+
+    def test_service_manager(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            service_manager.assert_within_limits()
+            assert_equal(service_manager.config.service_type, 'geocoder_mapzen')
+
+    def test_no_rate_limit_by_default(self):
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3, False)
+
+    def test_no_legacy_rate_limit_by_default(self):
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3, False)
+
+    def test_legacy_server_rate_limit(self):
+        rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_server_rate_limit(self):
+        rate_limits = '{"geocoder":{"limit":"3","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': rate_limits}])
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_user_rate_limit(self):
+        user_redis_name = "rails:users:{0}".format(self.username)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_user_rate_limit(self):
+        user_redis_name = "rails:users:{0}".format(self.username)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_org_rate_limit(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_org_rate_limit(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+
+    def test_user_rate_limit_precedence_over_org(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"1000","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        user_redis_name = "rails:users:{0}".format(self.username)
+        user_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_org_rate_limit_precedence_over_server(self):
+        server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        with patch.object(RedisConnectionBuilder,'get') as get_fn:
+            get_fn.return_value = self.redis_conn
+            service_manager = ServiceManager('geocoder', MapzenGeocoderConfigBuilder, self.username, self.orgname)
+            self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])
+
+    def test_legacy_user_rate_limit_precedence_over_org(self):
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"1000","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        user_redis_name = "rails:users:{0}".format(self.username)
+        user_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(user_redis_name, 'geocoder_rate_limit', user_rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        self.redis_conn.hdel(user_redis_name, 'geocoder_rate_limit')
+
+    def test_legacy_org_rate_limit_precedence_over_server(self):
+        server_rate_limits = '{"geocoder":{"limit":"1000","period":3600}}'
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [{'conf': server_rate_limits}])
+        org_redis_name = "rails:orgs:{0}".format(self.orgname)
+        org_rate_limits = '{"limit":"3","period":3600}'
+        self.redis_conn.hset(org_redis_name, 'geocoder_rate_limit', org_rate_limits)
+        config = GeocoderConfig(self.redis_conn, plpy_mock, self.username, self.orgname,  'mapzen')
+        config_cache = {
+          'redis_connection_test_user' : { 'redis_metrics_connection': self.redis_conn },
+          'user_geocoder_config_test_user' : config,
+          'logger_config' : Mock(min_log_level='debug', log_file_path=None, rollbar_api_key=None, environment=self.environment)
+        }
+        service_manager = LegacyServiceManager('geocoder', self.username, self.orgname, config_cache)
+        self.check_rate_limit(service_manager, 3)
+        self.redis_conn.hdel(org_redis_name, 'geocoder_rate_limit')
+        plpy_mock._define_result("CDB_Conf_GetConf\('rate_limits'\)", [])

server/lib/python/cartodb_services/test/test_user_service.py

@@ -90,19 +90,6 @@ class TestUserService(TestCase):
         self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 400)
         assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
 
-    @freeze_time("2015-06-01")
-    def test_should_account_for_wrongly_stored_non_padded_keys(self):
-        us = self.__build_user_service('test_user')
-        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '1', 400)
-        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 400
-
-    @freeze_time("2015-06-01")
-    def test_should_sum_amounts_from_both_key_formats(self):
-        us = self.__build_user_service('test_user')
-        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506',  '1', 400)
-        self.redis_conn.zincrby('user:test_user:geocoder_here:success_responses:201506', '01', 300)
-        assert us.used_quota(self.NOKIA_GEOCODER, date(2015, 6,1)) == 700
-
     @freeze_time("2015-06-15")
     def test_should_not_request_redis_twice_when_unneeded(self):
         class MockRedisWithCounter(MockRedis):