From 424564e3246e09c786d5135e44bee4e1b500e76f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 8 Jun 2015 13:37:27 -0400 Subject: [PATCH 1/6] initial commit --- scripts-available/CDB_DistType.sql | 142 +++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 scripts-available/CDB_DistType.sql diff --git a/scripts-available/CDB_DistType.sql b/scripts-available/CDB_DistType.sql new file mode 100644 index 0000000..07f685a --- /dev/null +++ b/scripts-available/CDB_DistType.sql @@ -0,0 +1,142 @@ +-- +-- CDB_DistType classifies the histograms of a column into +-- one of the basic types listed by Galtong: http://druedin.com/2012/12/08/galtungs-ajus-system/ +-- +-- Future improvements: +-- variable number of bins (7 is baked in right now) +-- catch the number of items to ensure that the sample is large enough +-- +-- Refs: +-- 1. width_bucket/histograms: http://tapoueh.org/blog/2014/02/21-PostgreSQL-histogram +-- 2. R implementation: https://github.com/cran/agrmt + + +CREATE OR REPLACE FUNCTION CDB_ClassTest ( in_array NUMERIC[] ) RETURNS text as $$ +DECLARE + element_count INT4; + minv numeric; + maxv numeric; + bins numeric[]; + freqs numeric[]; + diff numeric[]; + diffdiff numeric[]; + ajus INT[]; + freq INT4; + signature text; + i INT := 1; +BEGIN + SELECT min(e), max(e), count(e) INTO minv, maxv, element_count FROM ( SELECT unnest(in_array) e ) x; + + IF abs(maxv - minv) < 1e-7 THEN -- if max and min are nearly equal, call if 'F' + signature = 'F'; + ELSE + -- Calculate bins and count in bins + EXECUTE 'WITH stats as ( + SELECT min(e) as minv, + max(e) as maxv, + count(e) as total + FROM (SELECT unnest($1) e) x + WHERE e is not null + ), + hist as ( + SELECT width_bucket(e, s.minv, s.maxv, 6) bucket, + count(*) freq + FROM (SELECT unnest($1) e) x, stats s + WHERE e is not null + GROUP BY 1 + ORDER BY 1 + ) + SELECT array_agg(round(100.0 * hist.freq::numeric / stats.total::numeric,1)) freqs, + array_agg(hist.bucket) buckets + FROM hist, stats' + INTO freqs, bins + USING in_array; + -- don't need bins variable (freq are already sorted), discard it later + + LOOP + IF i < 7 THEN + ajus[i] = CDB_CompareValues(freqs[i],freqs[i+1],5.0); -- 5% tolerance + ELSE + EXIT; + END IF; + i := i + 1; + END LOOP; + + signature = CDB_DistributionType(ajus); + END IF; + + RETURN signature; +END; +$$ language plpgsql IMMUTABLE; + +-- Classify data into AJUSFL + +CREATE OR REPLACE FUNCTION CDB_DistributionType ( in_array INT[] ) RETURNS text as $$ +DECLARE + element_count INT4; + maxv numeric; + minv numeric; + uniques INT[]; + type text; +BEGIN + SELECT max(e), min(e) INTO maxv, minv FROM ( SELECT unnest(in_array) e ) x; + + IF (maxv = 0 AND minv = 0) THEN + type = 'F'; + ELSIF maxv < 1 THEN + type = 'L'; + ELSIF minv > -1 THEN + type = 'J'; + ELSE + -- Get distinct elements ordered by original position + EXECUTE 'WITH b AS ( + SELECT a + FROM (SELECT unnest($1) a) x + ), + c AS ( + SELECT a, row_number() OVER () r + FROM b + ), + d AS ( + SELECT DISTINCT a + FROM c + ), + e AS ( + SELECT a FROM d ORDER BY ( + SELECT r FROM c WHERE d.a = c.a ORDER BY r ASC LIMIT 1 + ) ASC) + SELECT array_agg(a) FROM e' + INTO uniques + USING in_array; + + -- Decide if it's an A, U, or other + IF (uniques = ARRAY[1,-1] OR uniques = ARRAY[1,0,-1] OR uniques = ARRAY[1,-1,0] OR uniques = ARRAY[0,1,-1]) THEN + type = 'A'; + ELSIF (uniques = ARRAY[-1,1] OR uniques = ARRAY[-1,0,1] OR uniques = ARRAY[-1,1,0] OR uniques = ARRAY[0,-1,1]) THEN + type = 'U'; + ELSE + type = 'S'; + END IF; + END IF; + + RETURN type; +END; +$$ language plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION CDB_CompareValues ( a numeric, b numeric, tolerance numeric ) RETURNS INT as $$ +DECLARE + d INT4; +BEGIN + IF a > b THEN + SELECT -1 INTO d; + ELSE + SELECT 1 INTO d; + END IF; + + IF abs(a-b) <= tolerance THEN + SELECT 0 INTO d; + END IF; + + RETURN d; +END; +$$ language plpgsql IMMUTABLE; \ No newline at end of file From 9cb1fe30d8f9cf750d0d369fbb57b320ff0440ee Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 8 Jun 2015 15:01:50 -0400 Subject: [PATCH 2/6] adding tests --- test/CDB_DistTypeTest.sql | 4 ++++ test/CDB_DistTypeTest_expect | 1 + 2 files changed, 5 insertions(+) create mode 100644 test/CDB_DistTypeTest.sql create mode 100644 test/CDB_DistTypeTest_expect diff --git a/test/CDB_DistTypeTest.sql b/test/CDB_DistTypeTest.sql new file mode 100644 index 0000000..efe1804 --- /dev/null +++ b/test/CDB_DistTypeTest.sql @@ -0,0 +1,4 @@ +WITH data AS ( + SELECT pow(x,3)::numeric x FROM generate_series(-100,100) x + ) +SELECT CDB_DistType(array_agg(x)) FROM data diff --git a/test/CDB_DistTypeTest_expect b/test/CDB_DistTypeTest_expect new file mode 100644 index 0000000..f70f10e --- /dev/null +++ b/test/CDB_DistTypeTest_expect @@ -0,0 +1 @@ +A From 8e2d86414f99bdc869e34a4267b7271a945d6e8e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 8 Jun 2015 15:02:39 -0400 Subject: [PATCH 3/6] updating function --- scripts-available/CDB_DistType.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts-available/CDB_DistType.sql b/scripts-available/CDB_DistType.sql index 07f685a..b60eb78 100644 --- a/scripts-available/CDB_DistType.sql +++ b/scripts-available/CDB_DistType.sql @@ -10,9 +10,8 @@ -- 1. width_bucket/histograms: http://tapoueh.org/blog/2014/02/21-PostgreSQL-histogram -- 2. R implementation: https://github.com/cran/agrmt - -CREATE OR REPLACE FUNCTION CDB_ClassTest ( in_array NUMERIC[] ) RETURNS text as $$ -DECLARE +CREATE OR REPLACE FUNCTION CDB_DistType ( in_array NUMERIC[] ) RETURNS text as $$ +DECLARE element_count INT4; minv numeric; maxv numeric; @@ -27,7 +26,7 @@ DECLARE BEGIN SELECT min(e), max(e), count(e) INTO minv, maxv, element_count FROM ( SELECT unnest(in_array) e ) x; - IF abs(maxv - minv) < 1e-7 THEN -- if max and min are nearly equal, call if 'F' + IF abs(maxv - minv) < 1e-7 THEN -- if max and min are nearly equal, call if 'F' (make relative to maxv?) signature = 'F'; ELSE -- Calculate bins and count in bins @@ -39,7 +38,7 @@ BEGIN WHERE e is not null ), hist as ( - SELECT width_bucket(e, s.minv, s.maxv, 6) bucket, + SELECT width_bucket(e, s.minv, s.maxv, 7) bucket, count(*) freq FROM (SELECT unnest($1) e) x, stats s WHERE e is not null From c7f4209270c80244acc9729e7bde0ade3f48100d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 8 Jun 2015 15:11:58 -0400 Subject: [PATCH 4/6] added alias and line --- scripts-available/CDB_DistType.sql | 2 +- scripts-enabled/CDB_DistType.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 scripts-enabled/CDB_DistType.sql diff --git a/scripts-available/CDB_DistType.sql b/scripts-available/CDB_DistType.sql index b60eb78..f74278d 100644 --- a/scripts-available/CDB_DistType.sql +++ b/scripts-available/CDB_DistType.sql @@ -138,4 +138,4 @@ BEGIN RETURN d; END; -$$ language plpgsql IMMUTABLE; \ No newline at end of file +$$ language plpgsql IMMUTABLE; diff --git a/scripts-enabled/CDB_DistType.sql b/scripts-enabled/CDB_DistType.sql new file mode 120000 index 0000000..64ff8c2 --- /dev/null +++ b/scripts-enabled/CDB_DistType.sql @@ -0,0 +1 @@ +../scripts-available/CDB_DistType.sql \ No newline at end of file From 6ab1b1d3d03075eb7c563e6a22690aa3e8e28c91 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 8 Jun 2015 18:36:07 -0400 Subject: [PATCH 5/6] removed unneeded variables --- scripts-available/CDB_DistType.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts-available/CDB_DistType.sql b/scripts-available/CDB_DistType.sql index f74278d..15dba49 100644 --- a/scripts-available/CDB_DistType.sql +++ b/scripts-available/CDB_DistType.sql @@ -17,8 +17,6 @@ DECLARE maxv numeric; bins numeric[]; freqs numeric[]; - diff numeric[]; - diffdiff numeric[]; ajus INT[]; freq INT4; signature text; @@ -50,7 +48,6 @@ BEGIN FROM hist, stats' INTO freqs, bins USING in_array; - -- don't need bins variable (freq are already sorted), discard it later LOOP IF i < 7 THEN From 1d223b77cc40118d0fcdbba3762718f3e71a883e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 10 Jun 2015 10:50:01 -0400 Subject: [PATCH 6/6] changed subfunction name, replaced function with case statement --- scripts-available/CDB_DistType.sql | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/scripts-available/CDB_DistType.sql b/scripts-available/CDB_DistType.sql index 15dba49..7d2242f 100644 --- a/scripts-available/CDB_DistType.sql +++ b/scripts-available/CDB_DistType.sql @@ -1,6 +1,6 @@ -- -- CDB_DistType classifies the histograms of a column into --- one of the basic types listed by Galtong: http://druedin.com/2012/12/08/galtungs-ajus-system/ +-- one of the basic types listed by Galtung: http://druedin.com/2012/12/08/galtungs-ajus-system/ -- -- Future improvements: -- variable number of bins (7 is baked in right now) @@ -51,14 +51,16 @@ BEGIN LOOP IF i < 7 THEN - ajus[i] = CDB_CompareValues(freqs[i],freqs[i+1],5.0); -- 5% tolerance + ajus[i] = CASE WHEN freqs[i] > freqs[i+1] THEN -1 + WHEN abs(freqs[i] - freqs[i+1]) <= 0.05 THEN 0 + ELSE 1 END; ELSE EXIT; END IF; i := i + 1; END LOOP; - signature = CDB_DistributionType(ajus); + signature = _CDB_DistTypeClassify(ajus); END IF; RETURN signature; @@ -67,7 +69,7 @@ $$ language plpgsql IMMUTABLE; -- Classify data into AJUSFL -CREATE OR REPLACE FUNCTION CDB_DistributionType ( in_array INT[] ) RETURNS text as $$ +CREATE OR REPLACE FUNCTION _CDB_DistTypeClassify ( in_array INT[] ) RETURNS text as $$ DECLARE element_count INT4; maxv numeric; @@ -118,21 +120,3 @@ BEGIN RETURN type; END; $$ language plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION CDB_CompareValues ( a numeric, b numeric, tolerance numeric ) RETURNS INT as $$ -DECLARE - d INT4; -BEGIN - IF a > b THEN - SELECT -1 INTO d; - ELSE - SELECT 1 INTO d; - END IF; - - IF abs(a-b) <= tolerance THEN - SELECT 0 INTO d; - END IF; - - RETURN d; -END; -$$ language plpgsql IMMUTABLE;