cartodb4/crankshaft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into update-segmentation

Browse Source
This commit is contained in:
Andy Eschbacher
2018-03-14 12:28:12 -04:00
parent 229e4671f7 8901b24eee
commit bc9cae1a63
189 changed files with 49373 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pg/crankshaft.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Spatial Analysis extension'
default_version = '0.6.1'
default_version = '0.8.1'
requires = 'plpythonu, postgis'
superuser = true
schema = cdb_crankshaft
src/pg/sql/10_moran.sql
+151 -9
View File
@@ -17,7 +17,7 @@ AS $$
num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (internal function)
-- Moran's I Local (internal function) - DEPRECATED
CREATE OR REPLACE FUNCTION
_CDB_AreasOfInterestLocal(
subquery TEXT,
@@ -27,16 +27,82 @@ CREATE OR REPLACE FUNCTION
permutations INT,
geom_col TEXT,
id_col TEXT)
RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
RETURNS TABLE (
moran NUMERIC,
quads TEXT,
significance NUMERIC,
rowid INT,
vals NUMERIC)
AS $$
from crankshaft.clustering import Moran
moran = Moran()
# TODO: use named parameters or a dictionary
return moran.local_stat(subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col)
result = moran.local_stat(subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col)
# remove spatial lag
return [(r[6], r[0], r[1], r[7], r[5]) for r in result]
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (internal function)
CREATE OR REPLACE FUNCTION
_CDB_MoransILocal(
subquery TEXT,
column_name TEXT,
w_type TEXT,
num_ngbrs INT,
permutations INT,
geom_col TEXT,
id_col TEXT)
RETURNS TABLE (
quads TEXT,
significance NUMERIC,
spatial_lag NUMERIC,
spatial_lag_std NUMERIC,
orig_val NUMERIC,
orig_val_std NUMERIC,
moran_stat NUMERIC,
rowid INT)
AS $$
from crankshaft.clustering import Moran
moran = Moran()
return moran.local_stat(subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (public-facing function)
-- Replaces CDB_AreasOfInterestLocal
CREATE OR REPLACE FUNCTION
CDB_MoransILocal(
subquery TEXT,
column_name TEXT,
w_type TEXT DEFAULT 'knn',
num_ngbrs INT DEFAULT 5,
permutations INT DEFAULT 99,
geom_col TEXT DEFAULT 'the_geom',
id_col TEXT DEFAULT 'cartodb_id')
RETURNS TABLE (
quads TEXT,
significance NUMERIC,
spatial_lag NUMERIC,
spatial_lag_std NUMERIC,
orig_val NUMERIC,
orig_val_std NUMERIC,
moran_stat NUMERIC,
rowid INT)
AS $$
SELECT
quads, significance, spatial_lag, spatial_lag_std,
orig_val, orig_val_std, moran_stat, rowid
FROM cdb_crankshaft._CDB_MoransILocal(
subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col);
$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (public-facing function) - DEPRECATED
CREATE OR REPLACE FUNCTION
CDB_AreasOfInterestLocal(
subquery TEXT,
@@ -132,7 +198,7 @@ AS $$
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Local Rate (internal function)
-- Moran's I Local Rate (internal function) - DEPRECATED
CREATE OR REPLACE FUNCTION
_CDB_AreasOfInterestLocalRate(
subquery TEXT,
@@ -144,15 +210,22 @@ CREATE OR REPLACE FUNCTION
geom_col TEXT,
id_col TEXT)
RETURNS
TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
TABLE(
moran NUMERIC,
quads TEXT,
significance NUMERIC,
rowid INT,
vals NUMERIC)
AS $$
from crankshaft.clustering import Moran
moran = Moran()
# TODO: use named parameters or a dictionary
return moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
result = moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
# remove spatial lag
return [(r[6], r[0], r[1], r[7], r[4]) for r in result]
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Local Rate (public-facing function)
-- Moran's I Local Rate (public-facing function) - DEPRECATED
CREATE OR REPLACE FUNCTION
CDB_AreasOfInterestLocalRate(
subquery TEXT,
@@ -172,6 +245,75 @@ AS $$
$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-- Internal function
CREATE OR REPLACE FUNCTION
_CDB_MoransILocalRate(
subquery TEXT,
numerator TEXT,
denominator TEXT,
w_type TEXT,
num_ngbrs INT,
permutations INT,
geom_col TEXT,
id_col TEXT)
RETURNS
TABLE(
quads TEXT,
significance NUMERIC,
spatial_lag NUMERIC,
spatial_lag_std NUMERIC,
orig_val NUMERIC,
orig_val_std NUMERIC,
moran_stat NUMERIC,
rowid INT)
AS $$
from crankshaft.clustering import Moran
moran = Moran()
return moran.local_rate_stat(
subquery,
numerator,
denominator,
w_type,
num_ngbrs,
permutations,
geom_col,
id_col
)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- Moran's I Rate
-- Replaces CDB_AreasOfInterestLocalRate
CREATE OR REPLACE FUNCTION
CDB_MoransILocalRate(
subquery TEXT,
numerator TEXT,
denominator TEXT,
w_type TEXT DEFAULT 'knn',
num_ngbrs INT DEFAULT 5,
permutations INT DEFAULT 99,
geom_col TEXT DEFAULT 'the_geom',
id_col TEXT DEFAULT 'cartodb_id')
RETURNS
TABLE(
quads TEXT,
significance NUMERIC,
spatial_lag NUMERIC,
spatial_lag_std NUMERIC,
orig_val NUMERIC,
orig_val_std NUMERIC,
moran_stat NUMERIC,
rowid INT)
AS $$
SELECT
quads, significance, spatial_lag, spatial_lag_std,
orig_val, orig_val_std, moran_stat, rowid
FROM cdb_crankshaft._CDB_MoransILocalRate(
subquery, numerator, denominator, w_type,
num_ngbrs, permutations, geom_col, id_col);
$$ LANGUAGE SQL VOLATILE PARALLEL UNSAFE;
-- Moran's I Local Rate only for HH and HL (public-facing function)
CREATE OR REPLACE FUNCTION
CDB_GetSpatialHotspotsRate(
src/pg/test/expected/02_moran_test.out
+114
View File
@@ -68,6 +68,63 @@ code|quads
(52 rows)
_cdb_random_seeds
(1 row)
code|quads|diff_orig|expected|moran_stat_not_null|significance_not_null|value_comparison
01|HH|t|t|t|t|t
02|HL|t|t|t|t|t
03|LL|t|t|t|t|t
04|LL|t|t|t|t|t
05|LH|t|t|t|t|t
06|LL|t|t|t|t|t
07|HH|t|t|t|t|t
08|HH|t|t|t|t|t
09|HH|t|t|t|t|t
10|LL|t|t|t|t|t
11|LL|t|t|t|t|t
12|LL|t|t|t|t|t
13|HL|t|t|t|t|t
14|LL|t|t|t|t|t
15|LL|t|t|t|t|t
16|HH|t|t|t|t|t
17|HH|t|t|t|t|t
18|LL|t|t|t|t|t
19|HH|t|t|t|t|t
20|HH|t|t|t|t|t
21|LL|t|t|t|t|t
22|HH|t|t|t|t|t
23|LL|t|t|t|t|t
24|LL|t|t|t|t|t
25|HH|t|t|t|t|t
26|HH|t|t|t|t|t
27|LL|t|t|t|t|t
28|HH|t|t|t|t|t
29|LL|t|t|t|t|t
30|LL|t|t|t|t|t
31|HH|t|t|t|t|t
32|LL|t|t|t|t|t
33|HL|t|t|t|t|t
34|LH|t|t|t|t|t
35|LL|t|t|t|t|t
36|LL|t|t|t|t|t
37|HL|t|t|t|t|t
38|HL|t|t|t|t|t
39|HH|t|t|t|t|t
40|HH|t|t|t|t|t
41|HL|t|t|t|t|t
42|LH|t|t|t|t|t
43|LH|t|t|t|t|t
44|LL|t|t|t|t|t
45|LH|t|t|t|t|t
46|LL|t|t|t|t|t
47|LL|t|t|t|t|t
48|HH|t|t|t|t|t
49|LH|t|t|t|t|t
50|HH|t|t|t|t|t
51|LL|t|t|t|t|t
52|LL|t|t|t|t|t
(52 rows)
_cdb_random_seeds
(1 row)
code|quads
01|HH
@@ -204,6 +261,63 @@ code|quads
(52 rows)
_cdb_random_seeds
(1 row)
code|quads|diff_orig|expected|moran_stat_not_null|significance_not_null
01|HH|t|t|t|t
02|HL|t|t|t|t
03|LL|t|t|t|t
04|LL|t|t|t|t
05|LH|t|t|t|t
06|LL|t|t|t|t
07|HH|t|t|t|t
08|HH|t|t|t|t
09|HH|t|t|t|t
10|LL|t|t|t|t
11|LL|t|t|t|t
12|LL|t|t|t|t
13|HL|t|t|t|t
14|LL|t|t|t|t
15|LL|t|t|t|t
16|HH|t|t|t|t
17|HH|t|t|t|t
18|LL|t|t|t|t
19|HH|t|t|t|t
20|HH|t|t|t|t
21|LL|t|t|t|t
22|HH|t|t|t|t
23|LL|t|t|t|t
24|LL|t|t|t|t
25|HH|t|t|t|t
26|HH|t|t|t|t
27|LL|t|t|t|t
28|HH|t|t|t|t
29|LL|t|t|t|t
30|LL|t|t|t|t
31|HH|t|t|t|t
32|LL|t|t|t|t
33|HL|t|t|t|t
34|LH|t|t|t|t
35|LL|t|t|t|t
36|LL|t|t|t|t
37|HL|t|t|t|t
38|HL|t|t|t|t
39|HH|t|t|t|t
40|HH|t|t|t|t
41|HL|t|t|t|t
42|LH|t|t|t|t
43|LH|t|t|t|t
44|LL|t|t|t|t
45|LH|t|t|t|t
46|LL|t|t|t|t
47|LL|t|t|t|t
48|HH|t|t|t|t
49|LH|t|t|t|t
50|HH|t|t|t|t
51|LL|t|t|t|t
52|LL|t|t|t|t
(52 rows)
_cdb_random_seeds
(1 row)
code|quads
01|HH
src/pg/test/sql/02_moran_test.sql
+37
View File
@@ -24,6 +24,25 @@ SELECT ppoints.code, m.quads
SELECT cdb_crankshaft._cdb_random_seeds(1234);
-- Moran's I local
SELECT
ppoints.code, m.quads,
abs(avg(m.orig_val_std) OVER ()) < 1e-6 as diff_orig,
CASE WHEN m.quads = 'HL' THEN m.orig_val_std > m.spatial_lag_std
WHEN m.quads = 'HH' THEN m.orig_val_std >= 0 and m.spatial_lag_std >= 0
WHEN m.quads = 'LH' THEN m.orig_val_std < m.spatial_lag_std
WHEN m.quads = 'LL' THEN m.orig_val_std <= 0 and m.spatial_lag_std <= 0
ELSE null END as expected,
moran_stat is not null moran_stat_not_null,
significance >= 0.001 significance_not_null, -- greater than 1/1000 (default)
abs(m.orig_val - ppoints.value) <= 1e-6 as value_comparison
FROM ppoints
JOIN cdb_crankshaft.CDB_MoransILocal('SELECT * FROM ppoints', 'value') m
ON ppoints.cartodb_id = m.rowid
ORDER BY ppoints.code;
SELECT cdb_crankshaft._cdb_random_seeds(1234);
-- Spatial Hotspots
SELECT ppoints.code, m.quads
FROM ppoints
@@ -61,6 +80,24 @@ SELECT ppoints2.code, m.quads
SELECT cdb_crankshaft._cdb_random_seeds(1234);
-- Moran's I local rate
SELECT
ppoints2.code, m.quads,
abs(avg(m.orig_val_std) OVER ()) < 1e-6 as diff_orig,
CASE WHEN m.quads = 'HL' THEN m.orig_val_std > m.spatial_lag_std
WHEN m.quads = 'HH' THEN m.orig_val_std >= 0 and m.spatial_lag_std >= 0
WHEN m.quads = 'LH' THEN m.orig_val_std < m.spatial_lag_std
WHEN m.quads = 'LL' THEN m.orig_val_std <= 0 and m.spatial_lag_std <= 0
ELSE null END as expected,
moran_stat is not null moran_stat_not_null,
significance >= 0.001 significance_not_null -- greater than 1/1000 (default)
FROM ppoints2
JOIN cdb_crankshaft.CDB_MoransILocalRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m
ON ppoints2.cartodb_id = m.rowid
ORDER BY ppoints2.code;
SELECT cdb_crankshaft._cdb_random_seeds(1234);
-- Spatial Hotspots (rate)
SELECT ppoints2.code, m.quads
FROM ppoints2