From fa6f9a8a661845e37dc857809c6513503f3f3769 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 11 Oct 2016 15:14:44 +0200 Subject: [PATCH 1/5] Implement CDB_CheckAnalysisQuota see #279 --- scripts-available/CDB_AnalysisCheck.sql | 50 +++++++++++++++++++ scripts-available/CDB_AnalysisSupport.sql | 55 +++++++++++++++++++++ scripts-enabled/270-CDB_AnalysisSupport.sql | 1 + scripts-enabled/275-CDB_AnalysisCheck.sql | 1 + 4 files changed, 107 insertions(+) create mode 100644 scripts-available/CDB_AnalysisCheck.sql create mode 100644 scripts-available/CDB_AnalysisSupport.sql create mode 120000 scripts-enabled/270-CDB_AnalysisSupport.sql create mode 120000 scripts-enabled/275-CDB_AnalysisCheck.sql diff --git a/scripts-available/CDB_AnalysisCheck.sql b/scripts-available/CDB_AnalysisCheck.sql new file mode 100644 index 0000000..ff30f2f --- /dev/null +++ b/scripts-available/CDB_AnalysisCheck.sql @@ -0,0 +1,50 @@ +-- Get the factor (fraction of the quota) for Camshaft cached analysis tables +CREATE OR REPLACE FUNCTION _CDB_AnalysisQuotaFactor() +RETURNS float8 AS +$$ +DECLARE + factor float8; +BEGIN + -- We use a floating point cdb_conf parameter + SELECT cdb_conf_getconf::text::float8 INTO factor FROM CDB_Conf_GetConf('analysis_cache_factor'); + -- With a default value + IF factor IS NULL THEN + factor := 0.2; + END IF; + RETURN factor; +END; +$$ +LANGUAGE 'plpgsql' STABLE; + +-- This checks the space used up by Camshaft cached analysis tables. +-- An exception will be raised if the limits are exceeded. +-- The name of an analysis table is passed; this, in addition to the +-- db role that executes this function is used to determined which +-- analysis tables will be considered. +CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT) +RETURNS void AS +$$ +DECLARE + schema_name TEXT; + user_name TEXT; + qmax int8; + cache_size float8; +BEGIN + -- We rely on the search_path to determine the user's schema and + -- check for all analysis tables in that schema. + -- An alternative would be to use cdb_analysis_catalog to + -- select analysis tables (cache_tables) from the same user, analysis or node. + -- For example: + -- SELECT unnest(cache_tables) FROM cdb_analysis_catalog + -- WHERE username IN (SELECT username FROM cdb_analysis_catalog + -- WHERE table_name::regclass = ANY (cache_tables)); + -- At the moment we're not using the provided table_name. + + SELECT current_schema() INTO schema_name; + EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax; + IF qmax*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN + -- The limit is defined by a factor applied to the total space quota for the user + RAISE EXCEPTION 'Analysis cache space limits exceeded'; + END IF; +END; +$$ LANGUAGE PLPGSQL; diff --git a/scripts-available/CDB_AnalysisSupport.sql b/scripts-available/CDB_AnalysisSupport.sql new file mode 100644 index 0000000..f1dd204 --- /dev/null +++ b/scripts-available/CDB_AnalysisSupport.sql @@ -0,0 +1,55 @@ +-- Internal auxiliar functions to deal with [Camshaft](https://github.com/cartodb/camshaft) cached analysis tables. + +-- This function returns TRUE if a given table name corresponds to a Camshaft cached analysis table +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_IsAnalysisTableName(table_name TEXT) +RETURNS BOOLEAN +AS $$ + BEGIN + RETURN table_name SIMILAR TO '\Aanalysis_[0-9a-f]{10}_[0-9a-f]{40}\Z'; + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- This function returns a relation of Camshaft cached analysis tables in the given schema. +-- If the schema name parameter is NULL, then tables from all schemas +-- that may contain user tables are returned. +-- For each table, the regclass, schema name and table name are returned. +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_AnalysisTablesInSchema(schema_name text DEFAULT NULL) +RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT) +AS $$ + SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE _CDB_IsAnalysisTableName(table_name); +$$ LANGUAGE 'sql'; + +-- This function returns a relation user tables excluding analysis tables +-- If the schema name parameter is NULL, then tables from all schemas +-- that may contain user tables are returned. +-- For each table, the regclass, schema name and table name are returned. +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_NonAnalysisTablesInSchema(schema_name text DEFAULT NULL) +RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT) +AS $$ + SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE Not _CDB_IsAnalysisTableName(table_name); +$$ LANGUAGE 'sql'; + +-- Total spaced used up by Camshaft cached analysis tables in the given schema. +-- Scope: private. +CREATE OR REPLACE FUNCTION _CDB_AnalysisDataSize(schema_name TEXT DEFAULT NULL) +RETURNS bigint AS +$$ +DECLARE + total_size INT8; +BEGIN + WITH analysis_tables AS ( + SELECT t.schema_name, t.table_name FROM _CDB_AnalysisTablesInSchema(schema_name) t + ) + SELECT COALESCE(INT8(SUM(_CDB_total_relation_size(analysis_tables.schema_name, analysis_tables.table_name))))::int8 + INTO total_size FROM analysis_tables; + IF total_size IS NOT NULL THEN + RETURN total_size; + ELSE + RETURN 0; + END IF; +END; +$$ +LANGUAGE 'plpgsql' VOLATILE; diff --git a/scripts-enabled/270-CDB_AnalysisSupport.sql b/scripts-enabled/270-CDB_AnalysisSupport.sql new file mode 120000 index 0000000..d586e3e --- /dev/null +++ b/scripts-enabled/270-CDB_AnalysisSupport.sql @@ -0,0 +1 @@ +../scripts-available/CDB_AnalysisSupport.sql \ No newline at end of file diff --git a/scripts-enabled/275-CDB_AnalysisCheck.sql b/scripts-enabled/275-CDB_AnalysisCheck.sql new file mode 120000 index 0000000..978dc35 --- /dev/null +++ b/scripts-enabled/275-CDB_AnalysisCheck.sql @@ -0,0 +1 @@ +../scripts-available/CDB_AnalysisCheck.sql \ No newline at end of file From 358c89b332dba7c1b7943bc995b045972600e595 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 11 Oct 2016 16:35:08 +0200 Subject: [PATCH 2/5] Add tests for CDB_CheckAnalysisQuota() --- test/CDB_AnalysisCheckTest.sql | 16 ++++++++++++++++ test/CDB_AnalysisCheckTest_expect | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 test/CDB_AnalysisCheckTest.sql create mode 100644 test/CDB_AnalysisCheckTest_expect diff --git a/test/CDB_AnalysisCheckTest.sql b/test/CDB_AnalysisCheckTest.sql new file mode 100644 index 0000000..c7490e0 --- /dev/null +++ b/test/CDB_AnalysisCheckTest.sql @@ -0,0 +1,16 @@ +SET client_min_messages TO error; +\set VERBOSITY terse + +SELECT CDB_SetUserQuotaInBytes(1000000); +SELECT _CDB_AnalysisTablesInSchema('public'); +SELECT _CDB_AnalysisDataSize('public'); +CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int); +CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int); +CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int); +SELECT _CDB_AnalysisTablesInSchema('public'); +SELECT _CDB_AnalysisDataSize('public'); +SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94'); +SELECT CDB_SetUserQuotaInBytes(1); +SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94'); +INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5); +SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94'); diff --git a/test/CDB_AnalysisCheckTest_expect b/test/CDB_AnalysisCheckTest_expect new file mode 100644 index 0000000..420a622 --- /dev/null +++ b/test/CDB_AnalysisCheckTest_expect @@ -0,0 +1,14 @@ +SET +1000000 +0 +CREATE TABLE +CREATE TABLE +CREATE TABLE +(analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5,public,analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5) +(analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94,public,analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94) +0 + +1 + +INSERT 0 5 +ERROR: Analysis cache space limits exceeded From 4786e0a2aeff68767803bbdc31e900626a552f53 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 11 Oct 2016 17:49:50 +0200 Subject: [PATCH 3/5] Test cleanup --- test/CDB_AnalysisCheckTest.sql | 4 ++++ test/CDB_AnalysisCheckTest_expect | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/test/CDB_AnalysisCheckTest.sql b/test/CDB_AnalysisCheckTest.sql index c7490e0..a1aa527 100644 --- a/test/CDB_AnalysisCheckTest.sql +++ b/test/CDB_AnalysisCheckTest.sql @@ -14,3 +14,7 @@ SELECT CDB_SetUserQuotaInBytes(1); SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94'); INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5); SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94'); +DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5; +DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94; +DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9; +DROP FUNCTION "public"._CDB_UserQuotaInBytes(); diff --git a/test/CDB_AnalysisCheckTest_expect b/test/CDB_AnalysisCheckTest_expect index 420a622..9769356 100644 --- a/test/CDB_AnalysisCheckTest_expect +++ b/test/CDB_AnalysisCheckTest_expect @@ -12,3 +12,7 @@ CREATE TABLE INSERT 0 5 ERROR: Analysis cache space limits exceeded +DROP TABLE +DROP TABLE +DROP TABLE +DROP FUNCTION From 066c574709a2b1eb6dc6d695bd90924afe6b5447 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 11 Oct 2016 17:52:11 +0200 Subject: [PATCH 4/5] Fix: cdb_conf table is not accessible to regular users --- scripts-available/CDB_AnalysisCheck.sql | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts-available/CDB_AnalysisCheck.sql b/scripts-available/CDB_AnalysisCheck.sql index ff30f2f..c8dddf3 100644 --- a/scripts-available/CDB_AnalysisCheck.sql +++ b/scripts-available/CDB_AnalysisCheck.sql @@ -1,3 +1,15 @@ +-- Read configuration parameter analysis_quota_factor, making it +-- accessible to regular users (which don't have access to cdb_conf) +CREATE OR REPLACE FUNCTION _CDB_GetConfAnalysisQuotaFactor() +RETURNS float8 AS +$$ +BEGIN + RETURN CDB_Conf_GetConf('analysis_quota_factor')::text::float8; +END; +$$ +LANGUAGE 'plpgsql' STABLE SECURITY DEFINER; + + -- Get the factor (fraction of the quota) for Camshaft cached analysis tables CREATE OR REPLACE FUNCTION _CDB_AnalysisQuotaFactor() RETURNS float8 AS @@ -6,7 +18,7 @@ DECLARE factor float8; BEGIN -- We use a floating point cdb_conf parameter - SELECT cdb_conf_getconf::text::float8 INTO factor FROM CDB_Conf_GetConf('analysis_cache_factor'); + factor := _CDB_GetConfAnalysisQuotaFactor(); -- With a default value IF factor IS NULL THEN factor := 0.2; From 529b12af20f710ad3cae7e936319d2771e46ae3c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 17 Oct 2016 17:16:04 +0200 Subject: [PATCH 5/5] Cosmetic fix: s/INT8/bigint #279 --- scripts-available/CDB_AnalysisSupport.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_AnalysisSupport.sql b/scripts-available/CDB_AnalysisSupport.sql index f1dd204..4b127da 100644 --- a/scripts-available/CDB_AnalysisSupport.sql +++ b/scripts-available/CDB_AnalysisSupport.sql @@ -38,7 +38,7 @@ CREATE OR REPLACE FUNCTION _CDB_AnalysisDataSize(schema_name TEXT DEFAULT NULL) RETURNS bigint AS $$ DECLARE - total_size INT8; + total_size bigint; BEGIN WITH analysis_tables AS ( SELECT t.schema_name, t.table_name FROM _CDB_AnalysisTablesInSchema(schema_name) t