From 457b614d961b68299067689ac3f2d7df2712690f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 5 Jul 2016 16:45:12 +0200 Subject: [PATCH 1/7] Exclude null geometries to create overviews Fixes #269 --- scripts-available/CDB_Overviews.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts-available/CDB_Overviews.sql b/scripts-available/CDB_Overviews.sql index 27743f1..d2ab5fd 100644 --- a/scripts-available/CDB_Overviews.sql +++ b/scripts-available/CDB_Overviews.sql @@ -687,6 +687,7 @@ AS $$ Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy, MIN(cartodb_id) AS cartodb_id FROM %1$s f + WHERE f.the_geom_webmercator IS NOT NULL GROUP BY gx, gy ) SELECT %6$s FROM clusters From 594543916db3b8fca11f4ab5d9f38b4528f23318 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 17 Oct 2016 18:03:00 +0200 Subject: [PATCH 2/7] New version 0.18.0 --- Makefile | 3 ++- NEWS.md | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d5979e9..7e568cc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.17.1 +EXTVERSION = 0.18.0 SED = sed @@ -73,6 +73,7 @@ UPGRADABLE = \ 0.16.4 \ 0.17.0 \ 0.17.1 \ + 0.18.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index f94c031..b721c4f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,11 @@ +0.18.0 (2016-10-17) +------------------- + +* Fix: exclude NULL geometries when creating Overviews #269 +* Function to check analysis tables limits #279 +* Exclude analysis cache tables from the quota #281 + + 0.17.1 (2016-08-16) ------------------- From 64fae71a371d8a571417681af79c26156664edd5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 19 Oct 2016 12:36:55 +0200 Subject: [PATCH 3/7] Default factor for analysis size from 0.2 to 2 Change the default value for the factor of analysis tables size from 0.2 to 2. I also checked it is applied on the "nominal" user quota. --- scripts-available/CDB_AnalysisCheck.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_AnalysisCheck.sql b/scripts-available/CDB_AnalysisCheck.sql index c8dddf3..7ad076f 100644 --- a/scripts-available/CDB_AnalysisCheck.sql +++ b/scripts-available/CDB_AnalysisCheck.sql @@ -21,7 +21,7 @@ BEGIN factor := _CDB_GetConfAnalysisQuotaFactor(); -- With a default value IF factor IS NULL THEN - factor := 0.2; + factor := 2; END IF; RETURN factor; END; From bc5e23b14319027f9e34fe62da28b9941b22318d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 19 Oct 2016 12:41:18 +0200 Subject: [PATCH 4/7] Replace `qmax` by `nominal_quota` Since the analysis quota factor can be greater than 1, `qmax` can be a misleading name. Thus the change in var name. --- scripts-available/CDB_AnalysisCheck.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts-available/CDB_AnalysisCheck.sql b/scripts-available/CDB_AnalysisCheck.sql index 7ad076f..1aab236 100644 --- a/scripts-available/CDB_AnalysisCheck.sql +++ b/scripts-available/CDB_AnalysisCheck.sql @@ -39,7 +39,7 @@ $$ DECLARE schema_name TEXT; user_name TEXT; - qmax int8; + nominal_quota int8; cache_size float8; BEGIN -- We rely on the search_path to determine the user's schema and @@ -53,8 +53,8 @@ BEGIN -- At the moment we're not using the provided table_name. SELECT current_schema() INTO schema_name; - EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO qmax; - IF qmax*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN + EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO nominal_quota; + IF nominal_quota*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN -- The limit is defined by a factor applied to the total space quota for the user RAISE EXCEPTION 'Analysis cache space limits exceeded'; END IF; From 89b2999a80fc20cdd945e34b7a00a66089fb76e4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 19 Oct 2016 13:00:56 +0200 Subject: [PATCH 5/7] Release 0.18.1 --- Makefile | 3 ++- NEWS.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7e568cc..98a909a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.18.0 +EXTVERSION = 0.18.1 SED = sed @@ -74,6 +74,7 @@ UPGRADABLE = \ 0.17.0 \ 0.17.1 \ 0.18.0 \ + 0.18.1 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index b721c4f..0a492e1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.18.1 (2016-10-19) +------------------- + +* Increase analysis limit factor to 2 [#284](https://github.com/CartoDB/cartodb-postgresql/pull/284) + 0.18.0 (2016-10-17) ------------------- From 0bfdeae147be87dff5ede85df7339bb4c33bf2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Thu, 20 Oct 2016 14:30:02 +0200 Subject: [PATCH 6/7] Move "username" column to the last position on analysis_catalog - fixes #276 Due to the way it was first implemented, the "username" column would be on a different position depending on if it was an extension upgrade or a fresh install. This caused problems with pg_dumping databases and restoring them. pg_dump does not include the extension source (so this table's schema is not included on the dump) but does include this contents, using a COPY without column names by default and failing due to the order difference. After this has run, all tables (whether updated or not) will have the "username" column on the last position. --- scripts-available/CDB_AnalysisCatalog.sql | 42 +++++++++++++++++------ 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/scripts-available/CDB_AnalysisCatalog.sql b/scripts-available/CDB_AnalysisCatalog.sql index 905a821..b670d21 100644 --- a/scripts-available/CDB_AnalysisCatalog.sql +++ b/scripts-available/CDB_AnalysisCatalog.sql @@ -1,8 +1,6 @@ -- Table to register analysis nodes from https://github.com/cartodb/camshaft CREATE TABLE IF NOT EXISTS cartodb.cdb_analysis_catalog ( - -- useful for multi account deployments - username text, -- md5 hex hash node_id char(40) CONSTRAINT cdb_analysis_catalog_pkey PRIMARY KEY, -- being json allows to do queries like analysis_def->>'type' = 'buffer' @@ -28,7 +26,9 @@ cartodb.cdb_analysis_catalog ( -- store error message for failures last_error_message text, -- cached tables involved in the analysis - cache_tables regclass[] NOT NULL DEFAULT '{}' + cache_tables regclass[] NOT NULL DEFAULT '{}', + -- useful for multi account deployments + username text ); -- This can only be called from an SQL script executed by CREATE EXTENSION @@ -38,14 +38,9 @@ BEGIN END $$; -DO $$ - BEGIN - BEGIN - ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username text; - EXCEPTION - WHEN duplicate_column THEN END; - END; -$$; +-- Migrations to add new columns from old versions. +-- IMPORTANT: Those columns will be added in order of creation. To be consistent +-- in column order, ensure that new columns are added at the end and in the same order. DO $$ BEGIN @@ -73,3 +68,28 @@ DO $$ WHEN duplicate_column THEN END; END; $$; + +DO $$ + BEGIN + BEGIN + ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username text; + EXCEPTION + WHEN duplicate_column THEN END; + END; +$$; + +-- We want the "username" column to be moved to the last position if it was on a position from other versions +-- see https://github.com/CartoDB/cartodb-postgresql/issues/276 +DO LANGUAGE 'plpgsql' $$ + DECLARE + column_index int; + BEGIN + SELECT ordinal_position FROM information_schema.columns WHERE table_name='cdb_analysis_catalog' AND table_schema='cartodb' AND column_name='username' INTO column_index; + IF column_index = 1 OR column_index = 10 THEN + ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username_final text; + UPDATE cartodb.cdb_analysis_catalog SET username_final = username; + ALTER TABLE cartodb.cdb_analysis_catalog DROP COLUMN username; + ALTER TABLE cartodb.cdb_analysis_catalog RENAME COLUMN username_final TO username; + END IF; + END; +$$; From 5c52e7564fbec694255d4177584f40324046c620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Thu, 20 Oct 2016 15:58:55 +0200 Subject: [PATCH 7/7] Release 0.18.2 --- Makefile | 3 ++- NEWS.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 98a909a..54a5833 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.18.1 +EXTVERSION = 0.18.2 SED = sed @@ -75,6 +75,7 @@ UPGRADABLE = \ 0.17.1 \ 0.18.0 \ 0.18.1 \ + 0.18.2 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END) diff --git a/NEWS.md b/NEWS.md index 0a492e1..6bac54f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +0.18.2 (2016-10-20) +------------------- + +* Fix: cleanup inconsistent position of `username` column in analysis catalog after upgrades + [#285](https://github.com/cartodb/cartodb-postgresql/pull/285) + 0.18.1 (2016-10-19) -------------------