Merge pull request #280 from CartoDB/279-check-analysis

Implement CDB_CheckAnalysisQuota

scripts-available/CDB_AnalysisCheck.sql

@@ -0,0 +1,62 @@
+-- Read configuration parameter analysis_quota_factor, making it
+-- accessible to regular users (which don't have access to cdb_conf)
+CREATE OR REPLACE FUNCTION _CDB_GetConfAnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+BEGIN
+  RETURN CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE SECURITY DEFINER;
+
+
+-- Get the factor (fraction of the quota) for Camshaft cached analysis tables
+CREATE OR REPLACE FUNCTION _CDB_AnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+DECLARE
+  factor float8;
+BEGIN
+  -- We use a floating point cdb_conf parameter
+  factor := _CDB_GetConfAnalysisQuotaFactor();
+  -- With a default value
+  IF factor IS NULL THEN
+    factor := 0.2;
+  END IF;
+  RETURN factor;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE;
+
+-- This checks the space used up by Camshaft cached analysis tables.
+-- An exception will be raised if the limits are exceeded.
+-- The name of an analysis table is passed; this, in addition to the
+-- db role that executes this function is used to determined which
+-- analysis tables will be considered.
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+DECLARE
+  schema_name TEXT;
+  user_name TEXT;
+  qmax int8;
+  cache_size float8;
+BEGIN
+  -- We rely on the search_path to determine the user's schema and
+  -- check for all analysis tables in that schema.
+  -- An alternative would be to use cdb_analysis_catalog to
+  -- select analysis tables (cache_tables) from the same user, analysis or node.
+  -- For example:
+  --   SELECT unnest(cache_tables) FROM cdb_analysis_catalog
+  --     WHERE username IN (SELECT username FROM cdb_analysis_catalog
+  --       WHERE table_name::regclass = ANY (cache_tables));
+  -- At the moment we're not using the provided table_name.
+
+  SELECT current_schema() INTO schema_name;
+  EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax;
+  IF qmax*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN
+    -- The limit is defined by a factor applied to the total space quota for the user
+    RAISE EXCEPTION 'Analysis cache space limits exceeded';
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_AnalysisSupport.sql

@@ -0,0 +1,55 @@
+-- Internal auxiliar functions to deal with [Camshaft](https://github.com/cartodb/camshaft) cached analysis tables.
+
+-- This function returns TRUE if a given table name corresponds to a Camshaft cached analysis table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsAnalysisTableName(table_name TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN table_name SIMILAR TO '\Aanalysis_[0-9a-f]{10}_[0-9a-f]{40}\Z';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- This function returns a relation of Camshaft cached analysis tables in the given schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- This function returns a relation user tables excluding analysis tables
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_NonAnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE Not _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- Total spaced used up by Camshaft cached analysis tables in the given schema.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisDataSize(schema_name TEXT DEFAULT NULL)
+RETURNS bigint AS
+$$
+DECLARE
+  total_size bigint;
+BEGIN
+  WITH analysis_tables AS (
+    SELECT t.schema_name, t.table_name FROM _CDB_AnalysisTablesInSchema(schema_name) t
+  )
+  SELECT COALESCE(INT8(SUM(_CDB_total_relation_size(analysis_tables.schema_name, analysis_tables.table_name))))::int8
+    INTO total_size FROM analysis_tables;
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;

scripts-enabled/270-CDB_AnalysisSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisSupport.sql

scripts-enabled/275-CDB_AnalysisCheck.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisCheck.sql

test/CDB_AnalysisCheckTest.sql

@@ -0,0 +1,20 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+SELECT CDB_SetUserQuotaInBytes(1000000);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+SELECT CDB_SetUserQuotaInBytes(1);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9;
+DROP FUNCTION "public"._CDB_UserQuotaInBytes();

test/CDB_AnalysisCheckTest_expect

@@ -0,0 +1,18 @@
+SET
+1000000
+0
+CREATE TABLE
+CREATE TABLE
+CREATE TABLE
+(analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5,public,analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5)
+(analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94,public,analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94)
+0
+
+1
+
+INSERT 0 5
+ERROR:  Analysis cache space limits exceeded
+DROP TABLE
+DROP TABLE
+DROP TABLE
+DROP FUNCTION