From 691b9a8312c2865817f23f464f7538142e8a411c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 5 Dec 2017 16:16:39 -0500 Subject: [PATCH 01/10] moves quantile bins to use postgres precentile functions --- scripts-available/CDB_QuantileBins.sql | 46 ++++++-------------------- test/CDB_QuantileBinsTest.sql | 18 ++++++---- test/CDB_QuantileBinsTest_expect | 26 +++++++++------ 3 files changed, 39 insertions(+), 51 deletions(-) diff --git a/scripts-available/CDB_QuantileBins.sql b/scripts-available/CDB_QuantileBins.sql index 20fa440..c4a7564 100644 --- a/scripts-available/CDB_QuantileBins.sql +++ b/scripts-available/CDB_QuantileBins.sql @@ -5,39 +5,15 @@ -- bins based on the Quantile method. -- -- @param breaks The number of bins you want to find. --- -- -CREATE OR REPLACE FUNCTION CDB_QuantileBins ( in_array NUMERIC[], breaks INT) RETURNS NUMERIC[] as $$ -DECLARE - element_count INT4; - break_size numeric; - tmp_val numeric; - i INT := 1; - reply numeric[]; -BEGIN - -- sort our values - SELECT array_agg(e) INTO in_array FROM (SELECT unnest(in_array) e ORDER BY e ASC) x; - -- get the total size of our data - element_count := array_length(in_array, 1); - break_size := element_count::numeric / breaks; - -- slice our bread - LOOP - IF i < breaks THEN - IF break_size * i % 1 > 0 THEN - SELECT e INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 1 OFFSET ceil(break_size * i) - 1) x; - ELSE - SELECT avg(e) INTO tmp_val FROM ( SELECT unnest(in_array) e LIMIT 2 OFFSET ceil(break_size * i) - 1 ) x; - END IF; - ELSIF i = breaks THEN - -- select the last value - SELECT max(e) INTO tmp_val FROM ( SELECT unnest(in_array) e ) x; - ELSE - EXIT; - END IF; - - reply = array_append(reply, tmp_val); - i := i+1; - END LOOP; - RETURN reply; -END; -$$ language plpgsql IMMUTABLE STRICT PARALLEL SAFE; +-- +CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int) +RETURNS numeric[] +AS $$ + SELECT array_agg(p) FROM ( + SELECT percentile_disc(idx::numeric / breaks::numeric) + WITHIN GROUP (ORDER BY x ASC) AS p + FROM generate_series(1, breaks) AS idx, unnest(in_array) AS x + GROUP BY idx + ) AS quantiles; +$$ language sql; diff --git a/test/CDB_QuantileBinsTest.sql b/test/CDB_QuantileBinsTest.sql index 4a02a0c..90b9f62 100644 --- a/test/CDB_QuantileBinsTest.sql +++ b/test/CDB_QuantileBinsTest.sql @@ -1,11 +1,17 @@ WITH data AS ( - SELECT array_agg(x::numeric) s FROM generate_series(1,100) x - WHERE x % 5 != 0 AND x % 7 != 0 + SELECT array_agg(x::numeric) AS s + FROM generate_series(0, 99) AS x ) -SELECT unnest(CDB_QuantileBins(s, 7)) FROM data; +SELECT unnest(CDB_QuantileBins(s, 10)) + FROM data; WITH data_nulls AS ( - SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x - WHERE x % 5 != 0 AND x % 7 != 0 + SELECT array_agg(x::numeric) + FROM ( + SELECT x FROM generate_series(0, 99) AS x + UNION ALL + SELECT null AS s FROM generate_series(1, 10) AS x + ) _wrap ) -SELECT unnest(CDB_QuantileBins(s, 7)) FROM data_nulls; +SELECT unnest(CDB_QuantileBins(s, 10)) + FROM data_nulls; diff --git a/test/CDB_QuantileBinsTest_expect b/test/CDB_QuantileBinsTest_expect index 9261ae3..fbb4b0e 100644 --- a/test/CDB_QuantileBinsTest_expect +++ b/test/CDB_QuantileBinsTest_expect @@ -1,14 +1,20 @@ -13 +9 +19 29 -43 -57 -71 -86 +39 +49 +59 +69 +79 +89 99 +9 +19 29 -57 -87 - - - +39 +49 +59 +69 +79 +89 99 From 2fe02d8154e7cb6ff61ac2ad41fef30c6d3b2dff Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 5 Dec 2017 16:24:11 -0500 Subject: [PATCH 02/10] fix alias of column --- test/CDB_QuantileBinsTest.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CDB_QuantileBinsTest.sql b/test/CDB_QuantileBinsTest.sql index 90b9f62..342122d 100644 --- a/test/CDB_QuantileBinsTest.sql +++ b/test/CDB_QuantileBinsTest.sql @@ -6,11 +6,11 @@ SELECT unnest(CDB_QuantileBins(s, 10)) FROM data; WITH data_nulls AS ( - SELECT array_agg(x::numeric) + SELECT array_agg(x::numeric) AS s FROM ( SELECT x FROM generate_series(0, 99) AS x UNION ALL - SELECT null AS s FROM generate_series(1, 10) AS x + SELECT null AS x FROM generate_series(1, 10) AS x ) _wrap ) SELECT unnest(CDB_QuantileBins(s, 10)) From 4b937de4151589b97457765a58072124baaf691b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 6 Dec 2017 08:10:09 -0500 Subject: [PATCH 03/10] updates JenksTests to reflect expectations without nulls --- test/CDB_JenksBinsTest.sql | 24 ++++++++++++++++-------- test/CDB_JenksBinsTest_expect | 24 ++++++++++-------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/test/CDB_JenksBinsTest.sql b/test/CDB_JenksBinsTest.sql index b8bfbaa..2cc7071 100644 --- a/test/CDB_JenksBinsTest.sql +++ b/test/CDB_JenksBinsTest.sql @@ -1,11 +1,19 @@ WITH data AS ( - SELECT array_agg(x::numeric) s FROM generate_series(1,300) x - WHERE x % 5 != 0 AND x % 7 != 0 - ) -SELECT unnest(CDB_JenksBins(s, 7)) FROM data; + SELECT Array[0.99, 1.0, 1.01, + 4.99, 5.01, + 10.01, 10.01, + 15.01, 14.99, + 20.1, 19.9]::numeric[] AS s +) +SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data; WITH data_nulls AS ( - SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,300) x - WHERE x % 5 != 0 AND x % 7 != 0 - ) -SELECT unnest(CDB_JenksBins(s, 7)) FROM data_nulls; + SELECT Array[0.99, 1.0, 1.01, + 4.99, 5.01, + null, null, + 10.01, 10.01, + 15.01, 14.99, + null, null, + 20.1, 19.9]::numeric[] AS s +) +SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data_nulls; diff --git a/test/CDB_JenksBinsTest_expect b/test/CDB_JenksBinsTest_expect index 496e6be..98227ed 100644 --- a/test/CDB_JenksBinsTest_expect +++ b/test/CDB_JenksBinsTest_expect @@ -1,14 +1,10 @@ -43 -86 -129 -172 -213 -257 -299 -37 -51 -97 -157 -213 -241 - +1 +5 +10 +15 +20 +1 +5 +10 +15 +20 From dbaf795a7926859ae9f4b02e562f0219be77b4a0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 6 Dec 2017 08:18:24 -0500 Subject: [PATCH 04/10] adjust expectation to reality instead of expectation (added TODOs) --- test/CDB_JenksBinsTest.sql | 4 ++++ test/CDB_JenksBinsTest_expect | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/CDB_JenksBinsTest.sql b/test/CDB_JenksBinsTest.sql index 2cc7071..11d4312 100644 --- a/test/CDB_JenksBinsTest.sql +++ b/test/CDB_JenksBinsTest.sql @@ -5,6 +5,8 @@ WITH data AS ( 15.01, 14.99, 20.1, 19.9]::numeric[] AS s ) +-- expectation is: 1, 5, 10, 15, 20 +-- TODO: fix cdb_jenksbins to match ^^ SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data; WITH data_nulls AS ( @@ -16,4 +18,6 @@ WITH data_nulls AS ( null, null, 20.1, 19.9]::numeric[] AS s ) +-- expectation is: 1, 5, 10, 15, 20 +-- TODO: fix cdb_jenksbins to match ^^ SELECT round(unnest(CDB_JenksBins(s, 5))) FROM data_nulls; diff --git a/test/CDB_JenksBinsTest_expect b/test/CDB_JenksBinsTest_expect index 98227ed..8e69822 100644 --- a/test/CDB_JenksBinsTest_expect +++ b/test/CDB_JenksBinsTest_expect @@ -1,10 +1,10 @@ 1 5 10 -15 +20 20 1 5 10 -15 20 + From 0467c075f731191d48e4bf65b99e41c44341eccd Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 8 Mar 2018 10:00:52 -0500 Subject: [PATCH 05/10] opts for array version of percentile_disc --- scripts-available/CDB_QuantileBins.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts-available/CDB_QuantileBins.sql b/scripts-available/CDB_QuantileBins.sql index c4a7564..57c4e36 100644 --- a/scripts-available/CDB_QuantileBins.sql +++ b/scripts-available/CDB_QuantileBins.sql @@ -10,10 +10,9 @@ CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int) RETURNS numeric[] AS $$ - SELECT array_agg(p) FROM ( - SELECT percentile_disc(idx::numeric / breaks::numeric) - WITHIN GROUP (ORDER BY x ASC) AS p - FROM generate_series(1, breaks) AS idx, unnest(in_array) AS x - GROUP BY idx - ) AS quantiles; + SELECT + percentile_disc(Array(generate_series(1, breaks) / breaks::numeric)) + WITHIN GROUP (ORDER BY x ASC) AS p + FROM + unnest(in_array) AS x; $$ language sql; From cf38d4cf25753349eebfb48a4f32b879c7966241 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 8 Mar 2018 10:14:08 -0500 Subject: [PATCH 06/10] oof include SELECT --- scripts-available/CDB_QuantileBins.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-available/CDB_QuantileBins.sql b/scripts-available/CDB_QuantileBins.sql index 57c4e36..bde4b8b 100644 --- a/scripts-available/CDB_QuantileBins.sql +++ b/scripts-available/CDB_QuantileBins.sql @@ -11,8 +11,8 @@ CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int) RETURNS numeric[] AS $$ SELECT - percentile_disc(Array(generate_series(1, breaks) / breaks::numeric)) + percentile_disc(SELECT Array(generate_series(1, breaks) / breaks::numeric)) WITHIN GROUP (ORDER BY x ASC) AS p FROM unnest(in_array) AS x; -$$ language sql; +$$ language SQL IMMUTABLE STRICT PARALLEL SAFE; From 39998a9c88e788d1982f468427531f7314e039ed Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 8 Mar 2018 10:21:43 -0500 Subject: [PATCH 07/10] fix syntax error oof oof --- scripts-available/CDB_QuantileBins.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-available/CDB_QuantileBins.sql b/scripts-available/CDB_QuantileBins.sql index bde4b8b..deaa79d 100644 --- a/scripts-available/CDB_QuantileBins.sql +++ b/scripts-available/CDB_QuantileBins.sql @@ -11,7 +11,7 @@ CREATE OR REPLACE FUNCTION CDB_QuantileBins(in_array numeric[], breaks int) RETURNS numeric[] AS $$ SELECT - percentile_disc(SELECT Array(generate_series(1, breaks) / breaks::numeric)) + percentile_disc(Array(SELECT generate_series(1, breaks) / breaks::numeric)) WITHIN GROUP (ORDER BY x ASC) AS p FROM unnest(in_array) AS x; From 6714feea8f50dccdec91a9063dadc61bbd8a26b0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 8 Mar 2018 10:38:03 -0500 Subject: [PATCH 08/10] updates news --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 961e9d2..d1b50a5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +0.21.1 (2018-03-08) +* Refactors CDB_QuantileBins to rely on PostgreSQL function `percentile_disc` #316 + 0.21.0 (2018-02-15) * Add optional parameter to limit the number of cells in grid-generation functions #322 * Fix: grant usage on cartodb_id sequence when sharing read write #323 From 17c872ed61889dc96b90b03a7520c0725f441466 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 8 Mar 2018 11:02:19 -0500 Subject: [PATCH 09/10] changes version number --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index d1b50a5..85797da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -0.21.1 (2018-03-08) +0.22.0 (TBD) * Refactors CDB_QuantileBins to rely on PostgreSQL function `percentile_disc` #316 0.21.0 (2018-02-15) From 0a6f263106380a5d050634f718d8a29e7dc28a47 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 14 Mar 2018 12:14:50 +0100 Subject: [PATCH 10/10] Bump version to 0.22.0 --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f31953..00d9b05 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # cartodb/Makefile EXTENSION = cartodb -EXTVERSION = 0.21.0 +EXTVERSION = 0.22.0 SED = sed AWK = awk @@ -85,6 +85,7 @@ UPGRADABLE = \ 0.19.2 \ 0.20.0 \ 0.21.0 \ + 0.22.0 \ $(EXTVERSION)dev \ $(EXTVERSION)next \ $(END)