From a451fb5b6ac94b2849cabd6a039d73a96786d7a6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 29 Aug 2016 15:50:19 -0400 Subject: [PATCH 01/48] minor ordering changes --- src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 4622925..a588795 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -58,9 +58,9 @@ def query_attr_select(params): else: ## if moran's analysis attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')] + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] - for idx, val in enumerate(sorted(attrs)): + for idx, val in enumerate(attrs): attr_string += template % {"col": params[val], "alias_num": idx + 1} return attr_string From 623613aa5cd08cb9ddefbcadfb5adf6cd49389d7 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 29 Aug 2016 16:46:49 -0400 Subject: [PATCH 02/48] adding ordered dict to tests --- src/py/crankshaft/test/test_pysal_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 171fdbc..393adf6 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -2,18 +2,19 @@ import unittest import crankshaft.pysal_utils as pu from crankshaft import random_seeds +from collections import OrderedDict class PysalUtilsTest(unittest.TestCase): """Testing class for utility functions related to PySAL integrations""" def setUp(self): - self.params = {"id_col": "cartodb_id", - "attr1": "andy", - "attr2": "jay_z", - "subquery": "SELECT * FROM a_list", - "geom_col": "the_geom", - "num_ngbrs": 321} + self.params = OrderedDict([("id_col", "cartodb_id"), + ("attr1": "andy"), + ("attr2": "jay_z"), + ("subquery": "SELECT * FROM a_list"), + ("geom_col": "the_geom"), + ("num_ngbrs": 321)]) self.params_array = {"id_col": "cartodb_id", "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], @@ -69,7 +70,7 @@ class PysalUtilsTest(unittest.TestCase): "i.\"jay_z\" IS NOT NULL AND " \ "i.\"jay_z\" <> 0 " \ "ORDER BY i.\"cartodb_id\" ASC;" - + ans_array = "SELECT i.\"cartodb_id\" As id, " \ "i.\"_2013_dec\"::numeric As attr1, " \ "i.\"_2014_jan\"::numeric As attr2, " \ From 622235d787faa179d57dc241602f28cc48386f0a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 29 Aug 2016 16:52:40 -0400 Subject: [PATCH 03/48] :P adding commas --- src/py/crankshaft/test/test_pysal_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 393adf6..82c38f1 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -10,11 +10,11 @@ class PysalUtilsTest(unittest.TestCase): def setUp(self): self.params = OrderedDict([("id_col", "cartodb_id"), - ("attr1": "andy"), - ("attr2": "jay_z"), - ("subquery": "SELECT * FROM a_list"), - ("geom_col": "the_geom"), - ("num_ngbrs": 321)]) + ("attr1", "andy"), + ("attr2", "jay_z"), + ("subquery", "SELECT * FROM a_list"), + ("geom_col", "the_geom"), + ("num_ngbrs", 321)]) self.params_array = {"id_col": "cartodb_id", "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], From 40481f128610bb2acf9ed0c3cd49d5792ae06b41 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 29 Aug 2016 17:10:58 -0400 Subject: [PATCH 04/48] adding more tests --- src/py/crankshaft/test/test_pysal_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 82c38f1..9af8522 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -16,6 +16,13 @@ class PysalUtilsTest(unittest.TestCase): ("geom_col", "the_geom"), ("num_ngbrs", 321)]) + self.params2 = OrderedDict([("id_col", "cartodb_id"), + ("attr1", "price"), + ("attr2", "sq_meters"), + ("subquery", "SELECT * FROM a_list"), + ("geom_col", "the_geom"), + ("num_ngbrs", 321)]) + self.params_array = {"id_col": "cartodb_id", "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], "subquery": "SELECT * FROM a_list", @@ -28,11 +35,15 @@ class PysalUtilsTest(unittest.TestCase): ans = "i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, " + ans2 = "i.\"price\"::numeric As attr1, " \ + "i.\"sq_meters\"::numeric As attr2, " + ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ "i.\"_2014_jan\"::numeric As attr2, " \ "i.\"_2014_feb\"::numeric As attr3, " self.assertEqual(pu.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params2), ans2) self.assertEqual(pu.query_attr_select(self.params_array), ans_array) def test_query_attr_where(self): From 44dc5811b58dba58e5674ddb24d8a1eae524655a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 1 Sep 2016 16:47:57 -0400 Subject: [PATCH 05/48] updating tests for query ordering error --- src/pg/test/expected/02_moran_test.out | 215 ++++++++++++------------- 1 file changed, 106 insertions(+), 109 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 3ba5bfd..03e842a 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -143,135 +143,132 @@ _cdb_random_seeds (1 row) code|quads -01|LL -02|LH -03|HH -04|HH -05|LL -06|HH -07|LL -08|LL -09|LL -10|HH -11|HH -12|HL -13|LL -14|HH +01|HH +02|HL +03|LL +04|LL +05|LH +06|LL +07|HH +08|HH +09|HH +10|LL +11|LL +12|LL +13|HL +14|LL 15|LL -16|LL -17|LL -18|LH -19|LL -20|LL -21|HH -22|LL -23|HL +16|HH +17|HH +18|LL +19|HH +20|HH +21|LL +22|HH +23|LL 24|LL -25|LL -26|LL +25|HH +26|HH 27|LL -28|LL -29|LH -30|HH -31|LL +28|HH +29|LL +30|LL +31|HH 32|LL -33|LL -34|LL -35|LH -36|HL -37|LH -38|LH -39|LL -40|LL -41|LH -42|HL -43|LL -44|HL -45|LL -46|HL +33|HL +34|LH +35|LL +36|LL +37|HL +38|HL +39|HH +40|HH +41|HL +42|LH +43|LH +44|LL +45|LH +46|LL 47|LL -48|LL -49|HL -50|LL -51|HH +48|HH +49|LL +50|HH +51|LL (51 rows) _cdb_random_seeds (1 row) code|quads -03|HH -04|HH -06|HH -10|HH -11|HH -12|HL -14|HH -21|HH -23|HL -30|HH -36|HL -42|HL -44|HL -46|HL -49|HL -51|HH -(16 rows) +01|HH +02|HL +07|HH +08|HH +09|HH +13|HL +16|HH +17|HH +19|HH +20|HH +22|HH +25|HH +26|HH +28|HH +31|HH +33|HL +37|HL +38|HL +39|HH +40|HH +41|HL +48|HH +50|HH +(23 rows) _cdb_random_seeds (1 row) code|quads -01|LL -02|LH -05|LL -07|LL -08|LL -09|LL -13|LL +03|LL +04|LL +05|LH +06|LL +10|LL +11|LL +12|LL +14|LL 15|LL -16|LL -17|LL -18|LH -19|LL -20|LL -22|LL +18|LL +21|LL +23|LL 24|LL -25|LL -26|LL 27|LL -28|LL -29|LH -31|LL +29|LL +30|LL 32|LL -33|LL -34|LL -35|LH -37|LH -38|LH -39|LL -40|LL -41|LH -43|LL -45|LL +34|LH +35|LL +36|LL +42|LH +43|LH +44|LL +45|LH +46|LL 47|LL -48|LL -50|LL -(35 rows) +49|LL +51|LL +(28 rows) _cdb_random_seeds (1 row) code|quads -02|LH -12|HL -18|LH -23|HL -29|LH -35|LH -36|HL -37|LH -38|LH -41|LH -42|HL -44|HL -46|HL -49|HL -(14 rows) +02|HL +05|LH +13|HL +33|HL +34|LH +37|HL +38|HL +41|HL +42|LH +43|LH +45|LH +(11 rows) From e29f6f2861b271bd04192d064ddc9411e553164d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 6 Sep 2016 09:23:39 -0400 Subject: [PATCH 06/48] add more comments --- src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index a588795..f0c73ba 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -44,6 +44,13 @@ def query_attr_select(params): Create portion of SELECT statement for attributes inolved in query. @param params: dict of information used in query (column names, table name, etc.) + Example: + OrderedDict([('attr1', 'numerator'), + ('attr2', 'denominator'), + ('subquery', 'SELECT * FROM interesting_data')]) + Output: + "i.\"numerator\"::numeric As attr1, " \ + "i.\"denominator\"::numeric As attr2, " """ attr_string = "" From 1148aa417ae5da2d7beaa5d8e0d120e3410095f3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 6 Sep 2016 09:23:59 -0400 Subject: [PATCH 07/48] additional test on alphabetical ordering --- src/py/crankshaft/test/test_pysal_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 9af8522..aad9b20 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -23,6 +23,13 @@ class PysalUtilsTest(unittest.TestCase): ("geom_col", "the_geom"), ("num_ngbrs", 321)]) + self.params3 = OrderedDict([("id_col", "cartodb_id"), + ("attr1", "numerator"), + ("attr2", "denominator"), + ("subquery", "SELECT * FROM pecan"), + ("geom_col", "the_geom"), + ("num_ngbrs", 321)]) + self.params_array = {"id_col": "cartodb_id", "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], "subquery": "SELECT * FROM a_list", @@ -38,12 +45,16 @@ class PysalUtilsTest(unittest.TestCase): ans2 = "i.\"price\"::numeric As attr1, " \ "i.\"sq_meters\"::numeric As attr2, " + ans3 = "i.\"numerator\"::numeric As attr1, " \ + "i.\"denominator\"::numeric As attr2, " + ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ "i.\"_2014_jan\"::numeric As attr2, " \ "i.\"_2014_feb\"::numeric As attr3, " self.assertEqual(pu.query_attr_select(self.params), ans) self.assertEqual(pu.query_attr_select(self.params2), ans2) + self.assertEqual(pu.query_attr_select(self.params3), ans3) self.assertEqual(pu.query_attr_select(self.params_array), ans_array) def test_query_attr_where(self): From 60f52633face6b4803e241d28657a4494601740e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 9 Sep 2016 11:11:32 -0400 Subject: [PATCH 08/48] adds hotspot/coldspot function --- src/pg/sql/16_getis.sql | 16 ++++++ .../crankshaft/clustering/__init__.py | 1 + .../crankshaft/crankshaft/clustering/getis.py | 51 +++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 src/pg/sql/16_getis.sql create mode 100644 src/py/crankshaft/crankshaft/clustering/getis.py diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql new file mode 100644 index 0000000..37f288b --- /dev/null +++ b/src/pg/sql/16_getis.sql @@ -0,0 +1,16 @@ +-- Getis-Ord's G +-- Hotspot/Coldspot Analysis tool +CREATE OR REPLACE FUNCTION + CDB_GetisOrdsG( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, rowid BIGINT) +AS $$ + from crankshaft.clustering import getis + return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; diff --git a/src/py/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py index ed34fe0..d9682fa 100644 --- a/src/py/crankshaft/crankshaft/clustering/__init__.py +++ b/src/py/crankshaft/crankshaft/clustering/__init__.py @@ -1,3 +1,4 @@ """Import all functions from for clustering""" from moran import * from kmeans import * +from getis import * diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py new file mode 100644 index 0000000..29a2c50 --- /dev/null +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -0,0 +1,51 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy +from collections import OrderedDict + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def getis_ord(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Getis-Ord's G + Implementation building neighbors with a PostGIS database and Getis-Ord's G + hotspot/coldspot analysis with PySAL. + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors if kNN is chosen + + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(3) + except plpy.SPIError, e: + plpy.error('Query failed: %s' % e) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + getis = ps.esda.getisord(attr_vals, weight, star=True) + + return zip(getis.z_sim, getis.p_sim, weight.id_order) From ccccf680662446974d7c71a14965baaac0b09d40 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 12 Sep 2016 11:39:21 -0400 Subject: [PATCH 09/48] fix module call --- src/pg/sql/16_getis.sql | 13 ++++++------- src/py/crankshaft/crankshaft/clustering/getis.py | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index 37f288b..23547f1 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -4,13 +4,12 @@ CREATE OR REPLACE FUNCTION CDB_GetisOrdsG( subquery TEXT, column_name TEXT, - w_type TEXT, - num_ngbrs INT, - permutations INT, - geom_col TEXT, - id_col TEXT) + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, rowid BIGINT) AS $$ - from crankshaft.clustering import getis - return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + from crankshaft.clustering import getis_ord + return getis_ord(subquery, column_name, w_type, num_ngbrs, geom_col, id_col) $$ LANGUAGE plpythonu; diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index 29a2c50..8cbbd56 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -15,7 +15,7 @@ import crankshaft.pysal_utils as pu # High level interface --------------------------------------- def getis_ord(subquery, attr, - w_type, num_ngbrs, permutations, geom_col, id_col): + w_type, num_ngbrs, geom_col, id_col): """ Getis-Ord's G Implementation building neighbors with a PostGIS database and Getis-Ord's G @@ -39,13 +39,13 @@ def getis_ord(subquery, attr, # if there are no neighbors, exit if len(result) == 0: return pu.empty_zipped_array(3) - except plpy.SPIError, e: - plpy.error('Query failed: %s' % e) + except plpy.SPIError, err: + plpy.error('Query failed: %s' % err) attr_vals = pu.get_attributes(result) weight = pu.get_weight(result, w_type, num_ngbrs) # calculate LISA values - getis = ps.esda.getisord(attr_vals, weight, star=True) + getis = ps.esda.getisord.G_Local(attr_vals, weight, star=True) return zip(getis.z_sim, getis.p_sim, weight.id_order) From ce4cc637aeacb2ccee24372496a3e7f4d8575a29 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 09:05:24 -0400 Subject: [PATCH 10/48] adding permutations to interface --- src/pg/sql/16_getis.sql | 5 ++-- .../crankshaft/crankshaft/clustering/getis.py | 24 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index 23547f1..dc3a25d 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -6,10 +6,11 @@ CREATE OR REPLACE FUNCTION column_name TEXT, w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') -RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, rowid BIGINT) +RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, p_z_sim NUMERIC, rowid BIGINT) AS $$ from crankshaft.clustering import getis_ord - return getis_ord(subquery, column_name, w_type, num_ngbrs, geom_col, id_col) + return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index 8cbbd56..1ee425a 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -1,10 +1,7 @@ """ -Moran's I geostatistics (global clustering & outliers presence) +Getis-Ord's G geostatistics (hotspot/coldspot analysis) """ -# TODO: Fill in local neighbors which have null/NoneType values with the -# average of the their neighborhood - import pysal as ps import plpy from collections import OrderedDict @@ -15,11 +12,11 @@ import crankshaft.pysal_utils as pu # High level interface --------------------------------------- def getis_ord(subquery, attr, - w_type, num_ngbrs, geom_col, id_col): + w_type, num_ngbrs, permutations, geom_col, id_col): """ - Getis-Ord's G - Implementation building neighbors with a PostGIS database and Getis-Ord's G - hotspot/coldspot analysis with PySAL. + Getis-Ord's G* + Implementation building neighbors with a PostGIS database and PySAL's Getis-Ord's G* + hotspot/coldspot module. Andy Eschbacher """ @@ -38,14 +35,17 @@ def getis_ord(subquery, attr, result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return pu.empty_zipped_array(3) + return pu.empty_zipped_array(4) except plpy.SPIError, err: plpy.error('Query failed: %s' % err) attr_vals = pu.get_attributes(result) + + ## build PySAL weight object weight = pu.get_weight(result, w_type, num_ngbrs) - # calculate LISA values - getis = ps.esda.getisord.G_Local(attr_vals, weight, star=True) + # calculate Getis-Ord's G* z- and p-values + getis = ps.esda.getisord.G_Local(attr_vals, weight, + star=True, permutations=permutations) - return zip(getis.z_sim, getis.p_sim, weight.id_order) + return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order) From b71152a884b354cf57552fc89861d472e767c104 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 09:06:09 -0400 Subject: [PATCH 11/48] adds fixtures and tests --- src/py/crankshaft/test/fixtures/getis.json | 1 + .../test/fixtures/neighbors_getis.json | 1 + .../crankshaft/test/test_clustering_getis.py | 38 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 src/py/crankshaft/test/fixtures/getis.json create mode 100644 src/py/crankshaft/test/fixtures/neighbors_getis.json create mode 100644 src/py/crankshaft/test/test_clustering_getis.py diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json new file mode 100644 index 0000000..f2f7d88 --- /dev/null +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -0,0 +1 @@ +[[0.43099999999999999, -0.4491160909028587, 0.32667395669425392], [0.021999999999999999, -0.93788225530474956, 0.17415246162388542], [0.44600000000000001, -0.4491160909028587, 0.32667395669425392], [0.216, -0.64751176059715954, 0.25865039275726565], [0.11700000000000001, -0.96347562762330119, 0.1676544447116961], [0.049000000000000002, -0.20152426147617952, 0.42014433032225806], [0.072999999999999995, -0.46619525535633877, 0.32053787260380839], [0.16300000000000001, -0.74976349152754251, 0.22669858024734424], [0.052999999999999999, -0.64751176059715954, 0.25865039275726565], [0.156, -0.56083197633324544, 0.28745604294545346], [0.29599999999999999, -0.24280805846929457, 0.40407704630238783], [0.29099999999999998, -0.26928692847374969, 0.39385444517601476], [0.44, -0.67722330498207772, 0.24913214171492104], [0.16500000000000001, -0.011684382726338413, 0.49533871177388633], [0.378, -0.56473075182761978, 0.28612845711175672], [0.042999999999999997, -0.99072103490325258, 0.1609109084310375], [0.45600000000000002, -0.28743876271061125, 0.38688819224593796], [0.156, 0.13295980219309725, 0.4471125856868039], [0.19500000000000001, 0.0063207047268819955, 0.49747842043259183], [0.0070000000000000001, -1.0415760797098768, 0.1488041317793396], [0.111, -0.36942477246818262, 0.35590556804466089], [0.22500000000000001, 0.17506551624725114, 0.43051407691606425], [0.26900000000000002, -0.44363455341924385, 0.32865340985373048], [0.33800000000000002, -0.43284736233536597, 0.33256283131987785], [0.189, -0.60291193010388022, 0.27328363953070622], [0.20799999999999999, 0.035856268747985663, 0.48569848295465989], [0.123, 0.089678570182411793, 0.4642713229152271], [0.0030000000000000001, -1.1057224660076166, 0.13442333147768415], [0.16800000000000001, 0.11677761660057258, 0.45351814086906583], [0.29999999999999999, -0.013808110669475293, 0.49449153588626749], [0.378, -0.23290337417697893, 0.40791821913691739], [0.41699999999999998, 0.22945579816510672, 0.40925733604961323], [0.002, 6.5207630666532674, 3.4975355944766306e-11], [0.46200000000000002, -0.14672827921301743, 0.44167324829783083], [0.126, 0.22908126100922532, 0.40940287888500482], [0.047, -1.0840432095769956, 0.13917282042962942], [0.017000000000000001, 4.0493273511601489, 2.5682527508119612e-05], [0.067000000000000004, -0.93058949009641589, 0.17603297655968531], [0.435, -0.39105578866749485, 0.34787799959221344], [0.0030000000000000001, 6.4632623511010854, 5.1234683162704187e-11], [0.098000000000000004, 0.29440403358766087, 0.38422459253128149], [0.38400000000000001, -0.42044605311189681, 0.33707981556243372], [0.38700000000000001, 0.056785871984068648, 0.47735788415038005], [0.36899999999999999, -0.15875022725747437, 0.43693283214176271], [0.20399999999999999, 0.14086541129997771, 0.44398813370614665], [0.014, 6.0345506436163996, 7.9702822119998018e-10], [0.13400000000000001, -0.85883340027064359, 0.19521621870003536], [0.14499999999999999, -0.93901580426004294, 0.17386131505073421], [0.029000000000000001, 2.052209368389633, 0.020074659704574338], [0.45300000000000001, -0.2873860843780221, 0.38690835753455055], [0.27200000000000002, 0.11238736962962483, 0.45525813472736054], [0.47599999999999998, -0.22667990193583551, 0.41033632813474463], [0.060999999999999999, -0.66318887096788159, 0.25360479911357758], [0.47599999999999998, -0.41451366242647014, 0.33924898169485895], [0.14499999999999999, -0.65907426867863195, 0.25492403906163263], [0.127, -0.56550838435631712, 0.28586401153505392], [0.311, -0.019571757025783815, 0.49219249707263191], [0.27700000000000002, -0.049735022936677371, 0.48016677335804769], [0.20899999999999999, 0.34564692985400752, 0.36480403618322277], [0.26200000000000001, -0.055301366746933138, 0.47794918666392716], [0.315, -0.11227385176628404, 0.45530313698210811], [0.39900000000000002, -0.23091105262392728, 0.4086919520030482], [0.38, -0.05314541432928175, 0.47880802357740249], [0.29499999999999998, -0.048726126739086051, 0.48056877723722968], [0.29999999999999999, -0.65725648028245498, 0.25550800720533962], [0.35499999999999998, -0.29942829331962884, 0.38230663846055501], [0.39000000000000001, -0.38409317960223854, 0.35045469320011735], [0.249, -0.52740757430541685, 0.29895529238584306], [0.246, 0.34740441796104815, 0.36414375737078841], [0.089999999999999997, 0.47786308046182885, 0.31637383012047438], [0.23400000000000001, 0.0047163311140149293, 0.49811846308566454], [0.307, -0.098930796747205507, 0.46059660828759452], [0.40500000000000003, -0.37158786444006753, 0.35509986295795459], [0.45800000000000002, 0.60261000309518942, 0.27338408212548115], [0.44600000000000001, 0.046168582994860159, 0.48158794144995187], [0.28499999999999998, -0.049232649167827866, 0.48036694626611731], [0.050000000000000003, 1.0035582996961303, 0.15779578142846606], [0.218, 0.52664582141189142, 0.29921978282283035]] diff --git a/src/py/crankshaft/test/fixtures/neighbors_getis.json b/src/py/crankshaft/test/fixtures/neighbors_getis.json new file mode 100644 index 0000000..61e0c0e --- /dev/null +++ b/src/py/crankshaft/test/fixtures/neighbors_getis.json @@ -0,0 +1 @@ +[{"neighbors": [2, 6, 5, 10, 3], "hr8893": 1.624458, "id": 1}, {"neighbors": [4, 7, 9, 14, 16], "hr8893": 2.2554919999999998, "id": 2}, {"neighbors": [6, 0, 3, 8, 10], "hr8893": 1.4678899999999999, "id": 3}, {"neighbors": [8, 12, 2, 4, 6], "hr8893": 2.4842559999999998, "id": 4}, {"neighbors": [9, 12, 1, 3, 8], "hr8893": 0.0, "id": 5}, {"neighbors": [11, 10, 0, 15, 6], "hr8893": 9.0486730000000009, "id": 6}, {"neighbors": [2, 10, 0, 8, 17], "hr8893": 6.0294889999999999, "id": 7}, {"neighbors": [14, 1, 22, 16, 9], "hr8893": 1.8003849999999999, "id": 8}, {"neighbors": [3, 12, 19, 2, 18], "hr8893": 4.581251, "id": 9}, {"neighbors": [4, 16, 12, 1, 20], "hr8893": 3.7906070000000001, "id": 10}, {"neighbors": [17, 6, 5, 15, 0], "hr8893": 1.4474359999999999, "id": 11}, {"neighbors": [15, 5, 13, 21, 27], "hr8893": 1.1919660000000001, "id": 12}, {"neighbors": [8, 19, 3, 9, 4], "hr8893": 0.0, "id": 13}, {"neighbors": [21, 11, 28, 27, 15], "hr8893": 1.608017, "id": 14}, {"neighbors": [7, 16, 22, 1, 29], "hr8893": 1.9498120000000001, "id": 15}, {"neighbors": [11, 27, 26, 5, 10], "hr8893": 0.74509000000000003, "id": 16}, {"neighbors": [25, 9, 14, 29, 20], "hr8893": 4.1733180000000001, "id": 17}, {"neighbors": [31, 10, 18, 26, 6], "hr8893": 3.7832520000000001, "id": 18}, {"neighbors": [32, 17, 23, 19, 8], "hr8893": 2.0851359999999999, "id": 19}, {"neighbors": [23, 12, 20, 8, 18], "hr8893": 2.1763020000000002, "id": 20}, {"neighbors": [25, 23, 19, 34, 9], "hr8893": 6.3093469999999998, "id": 21}, {"neighbors": [13, 28, 27, 11, 35], "hr8893": 10.855743, "id": 22}, {"neighbors": [30, 14, 29, 24, 7], "hr8893": 4.211354, "id": 23}, {"neighbors": [19, 20, 34, 39, 36], "hr8893": 0.80481000000000003, "id": 24}, {"neighbors": [30, 41, 22, 43, 52], "hr8893": 3.2153309999999999, "id": 25}, {"neighbors": [20, 33, 16, 34, 29], "hr8893": 2.8336640000000002, "id": 26}, {"neighbors": [38, 31, 27, 15, 17], "hr8893": 1.5920399999999999, "id": 27}, {"neighbors": [35, 15, 21, 28, 26], "hr8893": 1.5711580000000001, "id": 28}, {"neighbors": [21, 37, 35, 27, 13], "hr8893": 3.1275900000000001, "id": 29}, {"neighbors": [33, 22, 30, 42, 16], "hr8893": 4.4168960000000004, "id": 30}, {"neighbors": [43, 22, 24, 29, 41], "hr8893": 3.0174859999999999, "id": 31}, {"neighbors": [40, 17, 26, 32, 49], "hr8893": 9.9242450000000009, "id": 32}, {"neighbors": [45, 39, 18, 31, 23], "hr8893": 7.9739570000000004, "id": 33}, {"neighbors": [25, 29, 44, 42, 34], "hr8893": 5.0054639999999999, "id": 34}, {"neighbors": [36, 20, 25, 23, 39], "hr8893": 2.4638909999999998, "id": 35}, {"neighbors": [27, 46, 37, 28, 38], "hr8893": 0.0, "id": 36}, {"neighbors": [39, 34, 50, 48, 23], "hr8893": 7.377974, "id": 37}, {"neighbors": [47, 28, 35, 46, 21], "hr8893": 1.0038750000000001, "id": 38}, {"neighbors": [51, 26, 35, 40, 27], "hr8893": 3.1900469999999999, "id": 39}, {"neighbors": [36, 45, 48, 32, 23], "hr8893": 45.905405999999999, "id": 40}, {"neighbors": [49, 31, 38, 45, 57], "hr8893": 2.447597, "id": 41}, {"neighbors": [52, 43, 30, 24, 53], "hr8893": 1.2949580000000001, "id": 42}, {"neighbors": [43, 44, 33, 53, 29], "hr8893": 5.9330980000000002, "id": 43}, {"neighbors": [53, 42, 30, 41, 60], "hr8893": 4.1339969999999999, "id": 44}, {"neighbors": [33, 42, 59, 58, 34], "hr8893": 4.298311, "id": 45}, {"neighbors": [48, 39, 32, 56, 40], "hr8893": 27.483827000000002, "id": 46}, {"neighbors": [35, 55, 47, 54, 37], "hr8893": 0.96979099999999996, "id": 47}, {"neighbors": [37, 54, 46, 35, 55], "hr8893": 0.0, "id": 48}, {"neighbors": [45, 50, 39, 62, 56], "hr8893": 2.934466, "id": 49}, {"neighbors": [40, 57, 51, 56, 45], "hr8893": 4.4564269999999997, "id": 50}, {"neighbors": [48, 36, 63, 59, 39], "hr8893": 4.629264, "id": 51}, {"neighbors": [61, 38, 55, 49, 57], "hr8893": 4.9415329999999997, "id": 52}, {"neighbors": [41, 64, 53, 43, 60], "hr8893": 3.9900410000000002, "id": 53}, {"neighbors": [43, 60, 64, 41, 42], "hr8893": 2.064324, "id": 54}, {"neighbors": [47, 55, 46, 37, 35], "hr8893": 3.0402529999999999, "id": 55}, {"neighbors": [54, 46, 61, 51, 67], "hr8893": 3.905411, "id": 56}, {"neighbors": [66, 62, 48, 45, 57], "hr8893": 4.3328389999999999, "id": 57}, {"neighbors": [49, 65, 61, 56, 51], "hr8893": 3.8941110000000001, "id": 58}, {"neighbors": [68, 59, 60, 44, 42], "hr8893": 6.8287940000000003, "id": 59}, {"neighbors": [69, 58, 63, 50, 44], "hr8893": 3.2639469999999999, "id": 60}, {"neighbors": [53, 68, 64, 58, 43], "hr8893": 3.2821630000000002, "id": 61}, {"neighbors": [67, 51, 55, 57, 65], "hr8893": 3.2957619999999999, "id": 62}, {"neighbors": [63, 48, 56, 66, 70], "hr8893": 7.2496790000000004, "id": 63}, {"neighbors": [62, 70, 69, 59, 50], "hr8893": 3.041846, "id": 64}, {"neighbors": [60, 53, 52, 71, 41], "hr8893": 1.618018, "id": 65}, {"neighbors": [57, 72, 66, 67, 75], "hr8893": 4.9108010000000002, "id": 66}, {"neighbors": [56, 75, 62, 74, 65], "hr8893": 1.991457, "id": 67}, {"neighbors": [61, 72, 65, 55, 57], "hr8893": 3.1461920000000001, "id": 68}, {"neighbors": [60, 58, 76, 71, 73], "hr8893": 7.2666500000000003, "id": 69}, {"neighbors": [73, 63, 59, 70, 77], "hr8893": 3.1109040000000001, "id": 70}, {"neighbors": [74, 63, 77, 69, 62], "hr8893": 2.9802710000000001, "id": 71}, {"neighbors": [68, 64, 76, 60, 53], "hr8893": 3.8667669999999998, "id": 72}, {"neighbors": [65, 67, 75, 61, 57], "hr8893": 1.8684080000000001, "id": 73}, {"neighbors": [69, 76, 77, 68, 59], "hr8893": 12.577033999999999, "id": 74}, {"neighbors": [75, 70, 66, 77, 62], "hr8893": 7.8035990000000002, "id": 75}, {"neighbors": [74, 66, 72, 65, 70], "hr8893": 3.4714900000000002, "id": 76}, {"neighbors": [68, 73, 71, 69, 60], "hr8893": 4.334822, "id": 77}, {"neighbors": [70, 74, 69, 73, 63], "hr8893": 8.4515370000000001, "id": 78}] diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py new file mode 100644 index 0000000..fc26fc7 --- /dev/null +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class GetisTest(unittest.TestCase): + """Testing class for Getis-Ord's G funtion""" + + def setUp(self): + plpy._reset() + print(help(cc)) + self.neighbors_data = json.loads(open(fixture_file('neighbors_getis.json')).read()) + self.getis_data = json.loads(open(fixture_file('getis.json')).read()) + + def test_getis_ord(self): + """Test Getis-Ord's G*""" + data = [ { 'id': d['id'], + 'attr1': d['hr8893'], + 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.getis_ord('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.getis_data + for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) From 4e42625d7915919b835d4453df661c36f1561eed Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 14:29:06 -0400 Subject: [PATCH 12/48] fix indexing of fixture --- .../test/fixtures/neighbors_getis.json | 861 +++++++++++++++++- 1 file changed, 860 insertions(+), 1 deletion(-) diff --git a/src/py/crankshaft/test/fixtures/neighbors_getis.json b/src/py/crankshaft/test/fixtures/neighbors_getis.json index 61e0c0e..878b955 100644 --- a/src/py/crankshaft/test/fixtures/neighbors_getis.json +++ b/src/py/crankshaft/test/fixtures/neighbors_getis.json @@ -1 +1,860 @@ -[{"neighbors": [2, 6, 5, 10, 3], "hr8893": 1.624458, "id": 1}, {"neighbors": [4, 7, 9, 14, 16], "hr8893": 2.2554919999999998, "id": 2}, {"neighbors": [6, 0, 3, 8, 10], "hr8893": 1.4678899999999999, "id": 3}, {"neighbors": [8, 12, 2, 4, 6], "hr8893": 2.4842559999999998, "id": 4}, {"neighbors": [9, 12, 1, 3, 8], "hr8893": 0.0, "id": 5}, {"neighbors": [11, 10, 0, 15, 6], "hr8893": 9.0486730000000009, "id": 6}, {"neighbors": [2, 10, 0, 8, 17], "hr8893": 6.0294889999999999, "id": 7}, {"neighbors": [14, 1, 22, 16, 9], "hr8893": 1.8003849999999999, "id": 8}, {"neighbors": [3, 12, 19, 2, 18], "hr8893": 4.581251, "id": 9}, {"neighbors": [4, 16, 12, 1, 20], "hr8893": 3.7906070000000001, "id": 10}, {"neighbors": [17, 6, 5, 15, 0], "hr8893": 1.4474359999999999, "id": 11}, {"neighbors": [15, 5, 13, 21, 27], "hr8893": 1.1919660000000001, "id": 12}, {"neighbors": [8, 19, 3, 9, 4], "hr8893": 0.0, "id": 13}, {"neighbors": [21, 11, 28, 27, 15], "hr8893": 1.608017, "id": 14}, {"neighbors": [7, 16, 22, 1, 29], "hr8893": 1.9498120000000001, "id": 15}, {"neighbors": [11, 27, 26, 5, 10], "hr8893": 0.74509000000000003, "id": 16}, {"neighbors": [25, 9, 14, 29, 20], "hr8893": 4.1733180000000001, "id": 17}, {"neighbors": [31, 10, 18, 26, 6], "hr8893": 3.7832520000000001, "id": 18}, {"neighbors": [32, 17, 23, 19, 8], "hr8893": 2.0851359999999999, "id": 19}, {"neighbors": [23, 12, 20, 8, 18], "hr8893": 2.1763020000000002, "id": 20}, {"neighbors": [25, 23, 19, 34, 9], "hr8893": 6.3093469999999998, "id": 21}, {"neighbors": [13, 28, 27, 11, 35], "hr8893": 10.855743, "id": 22}, {"neighbors": [30, 14, 29, 24, 7], "hr8893": 4.211354, "id": 23}, {"neighbors": [19, 20, 34, 39, 36], "hr8893": 0.80481000000000003, "id": 24}, {"neighbors": [30, 41, 22, 43, 52], "hr8893": 3.2153309999999999, "id": 25}, {"neighbors": [20, 33, 16, 34, 29], "hr8893": 2.8336640000000002, "id": 26}, {"neighbors": [38, 31, 27, 15, 17], "hr8893": 1.5920399999999999, "id": 27}, {"neighbors": [35, 15, 21, 28, 26], "hr8893": 1.5711580000000001, "id": 28}, {"neighbors": [21, 37, 35, 27, 13], "hr8893": 3.1275900000000001, "id": 29}, {"neighbors": [33, 22, 30, 42, 16], "hr8893": 4.4168960000000004, "id": 30}, {"neighbors": [43, 22, 24, 29, 41], "hr8893": 3.0174859999999999, "id": 31}, {"neighbors": [40, 17, 26, 32, 49], "hr8893": 9.9242450000000009, "id": 32}, {"neighbors": [45, 39, 18, 31, 23], "hr8893": 7.9739570000000004, "id": 33}, {"neighbors": [25, 29, 44, 42, 34], "hr8893": 5.0054639999999999, "id": 34}, {"neighbors": [36, 20, 25, 23, 39], "hr8893": 2.4638909999999998, "id": 35}, {"neighbors": [27, 46, 37, 28, 38], "hr8893": 0.0, "id": 36}, {"neighbors": [39, 34, 50, 48, 23], "hr8893": 7.377974, "id": 37}, {"neighbors": [47, 28, 35, 46, 21], "hr8893": 1.0038750000000001, "id": 38}, {"neighbors": [51, 26, 35, 40, 27], "hr8893": 3.1900469999999999, "id": 39}, {"neighbors": [36, 45, 48, 32, 23], "hr8893": 45.905405999999999, "id": 40}, {"neighbors": [49, 31, 38, 45, 57], "hr8893": 2.447597, "id": 41}, {"neighbors": [52, 43, 30, 24, 53], "hr8893": 1.2949580000000001, "id": 42}, {"neighbors": [43, 44, 33, 53, 29], "hr8893": 5.9330980000000002, "id": 43}, {"neighbors": [53, 42, 30, 41, 60], "hr8893": 4.1339969999999999, "id": 44}, {"neighbors": [33, 42, 59, 58, 34], "hr8893": 4.298311, "id": 45}, {"neighbors": [48, 39, 32, 56, 40], "hr8893": 27.483827000000002, "id": 46}, {"neighbors": [35, 55, 47, 54, 37], "hr8893": 0.96979099999999996, "id": 47}, {"neighbors": [37, 54, 46, 35, 55], "hr8893": 0.0, "id": 48}, {"neighbors": [45, 50, 39, 62, 56], "hr8893": 2.934466, "id": 49}, {"neighbors": [40, 57, 51, 56, 45], "hr8893": 4.4564269999999997, "id": 50}, {"neighbors": [48, 36, 63, 59, 39], "hr8893": 4.629264, "id": 51}, {"neighbors": [61, 38, 55, 49, 57], "hr8893": 4.9415329999999997, "id": 52}, {"neighbors": [41, 64, 53, 43, 60], "hr8893": 3.9900410000000002, "id": 53}, {"neighbors": [43, 60, 64, 41, 42], "hr8893": 2.064324, "id": 54}, {"neighbors": [47, 55, 46, 37, 35], "hr8893": 3.0402529999999999, "id": 55}, {"neighbors": [54, 46, 61, 51, 67], "hr8893": 3.905411, "id": 56}, {"neighbors": [66, 62, 48, 45, 57], "hr8893": 4.3328389999999999, "id": 57}, {"neighbors": [49, 65, 61, 56, 51], "hr8893": 3.8941110000000001, "id": 58}, {"neighbors": [68, 59, 60, 44, 42], "hr8893": 6.8287940000000003, "id": 59}, {"neighbors": [69, 58, 63, 50, 44], "hr8893": 3.2639469999999999, "id": 60}, {"neighbors": [53, 68, 64, 58, 43], "hr8893": 3.2821630000000002, "id": 61}, {"neighbors": [67, 51, 55, 57, 65], "hr8893": 3.2957619999999999, "id": 62}, {"neighbors": [63, 48, 56, 66, 70], "hr8893": 7.2496790000000004, "id": 63}, {"neighbors": [62, 70, 69, 59, 50], "hr8893": 3.041846, "id": 64}, {"neighbors": [60, 53, 52, 71, 41], "hr8893": 1.618018, "id": 65}, {"neighbors": [57, 72, 66, 67, 75], "hr8893": 4.9108010000000002, "id": 66}, {"neighbors": [56, 75, 62, 74, 65], "hr8893": 1.991457, "id": 67}, {"neighbors": [61, 72, 65, 55, 57], "hr8893": 3.1461920000000001, "id": 68}, {"neighbors": [60, 58, 76, 71, 73], "hr8893": 7.2666500000000003, "id": 69}, {"neighbors": [73, 63, 59, 70, 77], "hr8893": 3.1109040000000001, "id": 70}, {"neighbors": [74, 63, 77, 69, 62], "hr8893": 2.9802710000000001, "id": 71}, {"neighbors": [68, 64, 76, 60, 53], "hr8893": 3.8667669999999998, "id": 72}, {"neighbors": [65, 67, 75, 61, 57], "hr8893": 1.8684080000000001, "id": 73}, {"neighbors": [69, 76, 77, 68, 59], "hr8893": 12.577033999999999, "id": 74}, {"neighbors": [75, 70, 66, 77, 62], "hr8893": 7.8035990000000002, "id": 75}, {"neighbors": [74, 66, 72, 65, 70], "hr8893": 3.4714900000000002, "id": 76}, {"neighbors": [68, 73, 71, 69, 60], "hr8893": 4.334822, "id": 77}, {"neighbors": [70, 74, 69, 73, 63], "hr8893": 8.4515370000000001, "id": 78}] +[ + { + "neighbors": [ + 2, + 6, + 5, + 10, + 3 + ], + "id": 0, + "value": 1.624458 + }, + { + "neighbors": [ + 4, + 7, + 9, + 14, + 16 + ], + "id": 1, + "value": 2.255492 + }, + { + "neighbors": [ + 6, + 0, + 3, + 8, + 10 + ], + "id": 2, + "value": 1.46789 + }, + { + "neighbors": [ + 8, + 12, + 2, + 4, + 6 + ], + "id": 3, + "value": 2.484256 + }, + { + "neighbors": [ + 9, + 12, + 1, + 3, + 8 + ], + "id": 4, + "value": 0 + }, + { + "neighbors": [ + 11, + 10, + 0, + 15, + 6 + ], + "id": 5, + "value": 9.048673 + }, + { + "neighbors": [ + 2, + 10, + 0, + 8, + 17 + ], + "id": 6, + "value": 6.029489 + }, + { + "neighbors": [ + 14, + 1, + 22, + 16, + 9 + ], + "id": 7, + "value": 1.800385 + }, + { + "neighbors": [ + 3, + 12, + 19, + 2, + 18 + ], + "id": 8, + "value": 4.581251 + }, + { + "neighbors": [ + 4, + 16, + 12, + 1, + 20 + ], + "id": 9, + "value": 3.790607 + }, + { + "neighbors": [ + 17, + 6, + 5, + 15, + 0 + ], + "id": 10, + "value": 1.447436 + }, + { + "neighbors": [ + 15, + 5, + 13, + 21, + 27 + ], + "id": 11, + "value": 1.191966 + }, + { + "neighbors": [ + 8, + 19, + 3, + 9, + 4 + ], + "id": 12, + "value": 0 + }, + { + "neighbors": [ + 21, + 11, + 28, + 27, + 15 + ], + "id": 13, + "value": 1.608017 + }, + { + "neighbors": [ + 7, + 16, + 22, + 1, + 29 + ], + "id": 14, + "value": 1.949812 + }, + { + "neighbors": [ + 11, + 27, + 26, + 5, + 10 + ], + "id": 15, + "value": 0.74509 + }, + { + "neighbors": [ + 25, + 9, + 14, + 29, + 20 + ], + "id": 16, + "value": 4.173318 + }, + { + "neighbors": [ + 31, + 10, + 18, + 26, + 6 + ], + "id": 17, + "value": 3.783252 + }, + { + "neighbors": [ + 32, + 17, + 23, + 19, + 8 + ], + "id": 18, + "value": 2.085136 + }, + { + "neighbors": [ + 23, + 12, + 20, + 8, + 18 + ], + "id": 19, + "value": 2.176302 + }, + { + "neighbors": [ + 25, + 23, + 19, + 34, + 9 + ], + "id": 20, + "value": 6.309347 + }, + { + "neighbors": [ + 13, + 28, + 27, + 11, + 35 + ], + "id": 21, + "value": 10.855743 + }, + { + "neighbors": [ + 30, + 14, + 29, + 24, + 7 + ], + "id": 22, + "value": 4.211354 + }, + { + "neighbors": [ + 19, + 20, + 34, + 39, + 36 + ], + "id": 23, + "value": 0.80481 + }, + { + "neighbors": [ + 30, + 41, + 22, + 43, + 52 + ], + "id": 24, + "value": 3.215331 + }, + { + "neighbors": [ + 20, + 33, + 16, + 34, + 29 + ], + "id": 25, + "value": 2.833664 + }, + { + "neighbors": [ + 38, + 31, + 27, + 15, + 17 + ], + "id": 26, + "value": 1.59204 + }, + { + "neighbors": [ + 35, + 15, + 21, + 28, + 26 + ], + "id": 27, + "value": 1.571158 + }, + { + "neighbors": [ + 21, + 37, + 35, + 27, + 13 + ], + "id": 28, + "value": 3.12759 + }, + { + "neighbors": [ + 33, + 22, + 30, + 42, + 16 + ], + "id": 29, + "value": 4.416896 + }, + { + "neighbors": [ + 43, + 22, + 24, + 29, + 41 + ], + "id": 30, + "value": 3.017486 + }, + { + "neighbors": [ + 40, + 17, + 26, + 32, + 49 + ], + "id": 31, + "value": 9.924245 + }, + { + "neighbors": [ + 45, + 39, + 18, + 31, + 23 + ], + "id": 32, + "value": 7.973957 + }, + { + "neighbors": [ + 25, + 29, + 44, + 42, + 34 + ], + "id": 33, + "value": 5.005464 + }, + { + "neighbors": [ + 36, + 20, + 25, + 23, + 39 + ], + "id": 34, + "value": 2.463891 + }, + { + "neighbors": [ + 27, + 46, + 37, + 28, + 38 + ], + "id": 35, + "value": 0 + }, + { + "neighbors": [ + 39, + 34, + 50, + 48, + 23 + ], + "id": 36, + "value": 7.377974 + }, + { + "neighbors": [ + 47, + 28, + 35, + 46, + 21 + ], + "id": 37, + "value": 1.003875 + }, + { + "neighbors": [ + 51, + 26, + 35, + 40, + 27 + ], + "id": 38, + "value": 3.190047 + }, + { + "neighbors": [ + 36, + 45, + 48, + 32, + 23 + ], + "id": 39, + "value": 45.905406 + }, + { + "neighbors": [ + 49, + 31, + 38, + 45, + 57 + ], + "id": 40, + "value": 2.447597 + }, + { + "neighbors": [ + 52, + 43, + 30, + 24, + 53 + ], + "id": 41, + "value": 1.294958 + }, + { + "neighbors": [ + 43, + 44, + 33, + 53, + 29 + ], + "id": 42, + "value": 5.933098 + }, + { + "neighbors": [ + 53, + 42, + 30, + 41, + 60 + ], + "id": 43, + "value": 4.133997 + }, + { + "neighbors": [ + 33, + 42, + 59, + 58, + 34 + ], + "id": 44, + "value": 4.298311 + }, + { + "neighbors": [ + 48, + 39, + 32, + 56, + 40 + ], + "id": 45, + "value": 27.483827 + }, + { + "neighbors": [ + 35, + 55, + 47, + 54, + 37 + ], + "id": 46, + "value": 0.969791 + }, + { + "neighbors": [ + 37, + 54, + 46, + 35, + 55 + ], + "id": 47, + "value": 0 + }, + { + "neighbors": [ + 45, + 50, + 39, + 62, + 56 + ], + "id": 48, + "value": 2.934466 + }, + { + "neighbors": [ + 40, + 57, + 51, + 56, + 45 + ], + "id": 49, + "value": 4.456427 + }, + { + "neighbors": [ + 48, + 36, + 63, + 59, + 39 + ], + "id": 50, + "value": 4.629264 + }, + { + "neighbors": [ + 61, + 38, + 55, + 49, + 57 + ], + "id": 51, + "value": 4.941533 + }, + { + "neighbors": [ + 41, + 64, + 53, + 43, + 60 + ], + "id": 52, + "value": 3.990041 + }, + { + "neighbors": [ + 43, + 60, + 64, + 41, + 42 + ], + "id": 53, + "value": 2.064324 + }, + { + "neighbors": [ + 47, + 55, + 46, + 37, + 35 + ], + "id": 54, + "value": 3.040253 + }, + { + "neighbors": [ + 54, + 46, + 61, + 51, + 67 + ], + "id": 55, + "value": 3.905411 + }, + { + "neighbors": [ + 66, + 62, + 48, + 45, + 57 + ], + "id": 56, + "value": 4.332839 + }, + { + "neighbors": [ + 49, + 65, + 61, + 56, + 51 + ], + "id": 57, + "value": 3.894111 + }, + { + "neighbors": [ + 68, + 59, + 60, + 44, + 42 + ], + "id": 58, + "value": 6.828794 + }, + { + "neighbors": [ + 69, + 58, + 63, + 50, + 44 + ], + "id": 59, + "value": 3.263947 + }, + { + "neighbors": [ + 53, + 68, + 64, + 58, + 43 + ], + "id": 60, + "value": 3.282163 + }, + { + "neighbors": [ + 67, + 51, + 55, + 57, + 65 + ], + "id": 61, + "value": 3.295762 + }, + { + "neighbors": [ + 63, + 48, + 56, + 66, + 70 + ], + "id": 62, + "value": 7.249679 + }, + { + "neighbors": [ + 62, + 70, + 69, + 59, + 50 + ], + "id": 63, + "value": 3.041846 + }, + { + "neighbors": [ + 60, + 53, + 52, + 71, + 41 + ], + "id": 64, + "value": 1.618018 + }, + { + "neighbors": [ + 57, + 72, + 66, + 67, + 75 + ], + "id": 65, + "value": 4.910801 + }, + { + "neighbors": [ + 56, + 75, + 62, + 74, + 65 + ], + "id": 66, + "value": 1.991457 + }, + { + "neighbors": [ + 61, + 72, + 65, + 55, + 57 + ], + "id": 67, + "value": 3.146192 + }, + { + "neighbors": [ + 60, + 58, + 76, + 71, + 73 + ], + "id": 68, + "value": 7.26665 + }, + { + "neighbors": [ + 73, + 63, + 59, + 70, + 77 + ], + "id": 69, + "value": 3.110904 + }, + { + "neighbors": [ + 74, + 63, + 77, + 69, + 62 + ], + "id": 70, + "value": 2.980271 + }, + { + "neighbors": [ + 68, + 64, + 76, + 60, + 53 + ], + "id": 71, + "value": 3.866767 + }, + { + "neighbors": [ + 65, + 67, + 75, + 61, + 57 + ], + "id": 72, + "value": 1.868408 + }, + { + "neighbors": [ + 69, + 76, + 77, + 68, + 59 + ], + "id": 73, + "value": 12.577034 + }, + { + "neighbors": [ + 75, + 70, + 66, + 77, + 62 + ], + "id": 74, + "value": 7.803599 + }, + { + "neighbors": [ + 74, + 66, + 72, + 65, + 70 + ], + "id": 75, + "value": 3.47149 + }, + { + "neighbors": [ + 68, + 73, + 71, + 69, + 60 + ], + "id": 76, + "value": 4.334822 + }, + { + "neighbors": [ + 70, + 74, + 69, + 73, + 63 + ], + "id": 77, + "value": 8.451537 + } +] From c392aec98a48ff9c56e05e238b1287fef3e7f19b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 15:32:32 -0400 Subject: [PATCH 13/48] re-ordered columns --- src/py/crankshaft/test/fixtures/getis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json index f2f7d88..20fe1a9 100644 --- a/src/py/crankshaft/test/fixtures/getis.json +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -1 +1 @@ -[[0.43099999999999999, -0.4491160909028587, 0.32667395669425392], [0.021999999999999999, -0.93788225530474956, 0.17415246162388542], [0.44600000000000001, -0.4491160909028587, 0.32667395669425392], [0.216, -0.64751176059715954, 0.25865039275726565], [0.11700000000000001, -0.96347562762330119, 0.1676544447116961], [0.049000000000000002, -0.20152426147617952, 0.42014433032225806], [0.072999999999999995, -0.46619525535633877, 0.32053787260380839], [0.16300000000000001, -0.74976349152754251, 0.22669858024734424], [0.052999999999999999, -0.64751176059715954, 0.25865039275726565], [0.156, -0.56083197633324544, 0.28745604294545346], [0.29599999999999999, -0.24280805846929457, 0.40407704630238783], [0.29099999999999998, -0.26928692847374969, 0.39385444517601476], [0.44, -0.67722330498207772, 0.24913214171492104], [0.16500000000000001, -0.011684382726338413, 0.49533871177388633], [0.378, -0.56473075182761978, 0.28612845711175672], [0.042999999999999997, -0.99072103490325258, 0.1609109084310375], [0.45600000000000002, -0.28743876271061125, 0.38688819224593796], [0.156, 0.13295980219309725, 0.4471125856868039], [0.19500000000000001, 0.0063207047268819955, 0.49747842043259183], [0.0070000000000000001, -1.0415760797098768, 0.1488041317793396], [0.111, -0.36942477246818262, 0.35590556804466089], [0.22500000000000001, 0.17506551624725114, 0.43051407691606425], [0.26900000000000002, -0.44363455341924385, 0.32865340985373048], [0.33800000000000002, -0.43284736233536597, 0.33256283131987785], [0.189, -0.60291193010388022, 0.27328363953070622], [0.20799999999999999, 0.035856268747985663, 0.48569848295465989], [0.123, 0.089678570182411793, 0.4642713229152271], [0.0030000000000000001, -1.1057224660076166, 0.13442333147768415], [0.16800000000000001, 0.11677761660057258, 0.45351814086906583], [0.29999999999999999, -0.013808110669475293, 0.49449153588626749], [0.378, -0.23290337417697893, 0.40791821913691739], [0.41699999999999998, 0.22945579816510672, 0.40925733604961323], [0.002, 6.5207630666532674, 3.4975355944766306e-11], [0.46200000000000002, -0.14672827921301743, 0.44167324829783083], [0.126, 0.22908126100922532, 0.40940287888500482], [0.047, -1.0840432095769956, 0.13917282042962942], [0.017000000000000001, 4.0493273511601489, 2.5682527508119612e-05], [0.067000000000000004, -0.93058949009641589, 0.17603297655968531], [0.435, -0.39105578866749485, 0.34787799959221344], [0.0030000000000000001, 6.4632623511010854, 5.1234683162704187e-11], [0.098000000000000004, 0.29440403358766087, 0.38422459253128149], [0.38400000000000001, -0.42044605311189681, 0.33707981556243372], [0.38700000000000001, 0.056785871984068648, 0.47735788415038005], [0.36899999999999999, -0.15875022725747437, 0.43693283214176271], [0.20399999999999999, 0.14086541129997771, 0.44398813370614665], [0.014, 6.0345506436163996, 7.9702822119998018e-10], [0.13400000000000001, -0.85883340027064359, 0.19521621870003536], [0.14499999999999999, -0.93901580426004294, 0.17386131505073421], [0.029000000000000001, 2.052209368389633, 0.020074659704574338], [0.45300000000000001, -0.2873860843780221, 0.38690835753455055], [0.27200000000000002, 0.11238736962962483, 0.45525813472736054], [0.47599999999999998, -0.22667990193583551, 0.41033632813474463], [0.060999999999999999, -0.66318887096788159, 0.25360479911357758], [0.47599999999999998, -0.41451366242647014, 0.33924898169485895], [0.14499999999999999, -0.65907426867863195, 0.25492403906163263], [0.127, -0.56550838435631712, 0.28586401153505392], [0.311, -0.019571757025783815, 0.49219249707263191], [0.27700000000000002, -0.049735022936677371, 0.48016677335804769], [0.20899999999999999, 0.34564692985400752, 0.36480403618322277], [0.26200000000000001, -0.055301366746933138, 0.47794918666392716], [0.315, -0.11227385176628404, 0.45530313698210811], [0.39900000000000002, -0.23091105262392728, 0.4086919520030482], [0.38, -0.05314541432928175, 0.47880802357740249], [0.29499999999999998, -0.048726126739086051, 0.48056877723722968], [0.29999999999999999, -0.65725648028245498, 0.25550800720533962], [0.35499999999999998, -0.29942829331962884, 0.38230663846055501], [0.39000000000000001, -0.38409317960223854, 0.35045469320011735], [0.249, -0.52740757430541685, 0.29895529238584306], [0.246, 0.34740441796104815, 0.36414375737078841], [0.089999999999999997, 0.47786308046182885, 0.31637383012047438], [0.23400000000000001, 0.0047163311140149293, 0.49811846308566454], [0.307, -0.098930796747205507, 0.46059660828759452], [0.40500000000000003, -0.37158786444006753, 0.35509986295795459], [0.45800000000000002, 0.60261000309518942, 0.27338408212548115], [0.44600000000000001, 0.046168582994860159, 0.48158794144995187], [0.28499999999999998, -0.049232649167827866, 0.48036694626611731], [0.050000000000000003, 1.0035582996961303, 0.15779578142846606], [0.218, 0.52664582141189142, 0.29921978282283035]] +[[-0.4491160909028587, 0.43099999999999999, 0.32667395669425392], [-0.93788225530474956, 0.021999999999999999, 0.17415246162388542], [-0.4491160909028587, 0.44600000000000001, 0.32667395669425392], [-0.64751176059715954, 0.216, 0.25865039275726565], [-0.96347562762330119, 0.11700000000000001, 0.1676544447116961], [-0.20152426147617952, 0.049000000000000002, 0.42014433032225806], [-0.46619525535633877, 0.072999999999999995, 0.32053787260380839], [-0.74976349152754251, 0.16300000000000001, 0.22669858024734424], [-0.64751176059715954, 0.052999999999999999, 0.25865039275726565], [-0.56083197633324544, 0.156, 0.28745604294545346], [-0.24280805846929457, 0.29599999999999999, 0.40407704630238783], [-0.26928692847374969, 0.29099999999999998, 0.39385444517601476], [-0.67722330498207772, 0.44, 0.24913214171492104], [-0.011684382726338413, 0.16500000000000001, 0.49533871177388633], [-0.56473075182761978, 0.378, 0.28612845711175672], [-0.99072103490325258, 0.042999999999999997, 0.1609109084310375], [-0.28743876271061125, 0.45600000000000002, 0.38688819224593796], [0.13295980219309725, 0.156, 0.4471125856868039], [0.0063207047268819955, 0.19500000000000001, 0.49747842043259183], [-1.0415760797098768, 0.0070000000000000001, 0.1488041317793396], [-0.36942477246818262, 0.111, 0.35590556804466089], [0.17506551624725114, 0.22500000000000001, 0.43051407691606425], [-0.44363455341924385, 0.26900000000000002, 0.32865340985373048], [-0.43284736233536597, 0.33800000000000002, 0.33256283131987785], [-0.60291193010388022, 0.189, 0.27328363953070622], [0.035856268747985663, 0.20799999999999999, 0.48569848295465989], [0.089678570182411793, 0.123, 0.4642713229152271], [-1.1057224660076166, 0.0030000000000000001, 0.13442333147768415], [0.11677761660057258, 0.16800000000000001, 0.45351814086906583], [-0.013808110669475293, 0.29999999999999999, 0.49449153588626749], [-0.23290337417697893, 0.378, 0.40791821913691739], [0.22945579816510672, 0.41699999999999998, 0.40925733604961323], [6.5207630666532674, 0.002, 3.4975355944766306e-11], [-0.14672827921301743, 0.46200000000000002, 0.44167324829783083], [0.22908126100922532, 0.126, 0.40940287888500482], [-1.0840432095769956, 0.047, 0.13917282042962942], [4.0493273511601489, 0.017000000000000001, 2.5682527508119612e-05], [-0.93058949009641589, 0.067000000000000004, 0.17603297655968531], [-0.39105578866749485, 0.435, 0.34787799959221344], [6.4632623511010854, 0.0030000000000000001, 5.1234683162704187e-11], [0.29440403358766087, 0.098000000000000004, 0.38422459253128149], [-0.42044605311189681, 0.38400000000000001, 0.33707981556243372], [0.056785871984068648, 0.38700000000000001, 0.47735788415038005], [-0.15875022725747437, 0.36899999999999999, 0.43693283214176271], [0.14086541129997771, 0.20399999999999999, 0.44398813370614665], [6.0345506436163996, 0.014, 7.9702822119998018e-10], [-0.85883340027064359, 0.13400000000000001, 0.19521621870003536], [-0.93901580426004294, 0.14499999999999999, 0.17386131505073421], [2.052209368389633, 0.029000000000000001, 0.020074659704574338], [-0.2873860843780221, 0.45300000000000001, 0.38690835753455055], [0.11238736962962483, 0.27200000000000002, 0.45525813472736054], [-0.22667990193583551, 0.47599999999999998, 0.41033632813474463], [-0.66318887096788159, 0.060999999999999999, 0.25360479911357758], [-0.41451366242647014, 0.47599999999999998, 0.33924898169485895], [-0.65907426867863195, 0.14499999999999999, 0.25492403906163263], [-0.56550838435631712, 0.127, 0.28586401153505392], [-0.019571757025783815, 0.311, 0.49219249707263191], [-0.049735022936677371, 0.27700000000000002, 0.48016677335804769], [0.34564692985400752, 0.20899999999999999, 0.36480403618322277], [-0.055301366746933138, 0.26200000000000001, 0.47794918666392716], [-0.11227385176628404, 0.315, 0.45530313698210811], [-0.23091105262392728, 0.39900000000000002, 0.4086919520030482], [-0.05314541432928175, 0.38, 0.47880802357740249], [-0.048726126739086051, 0.29499999999999998, 0.48056877723722968], [-0.65725648028245498, 0.29999999999999999, 0.25550800720533962], [-0.29942829331962884, 0.35499999999999998, 0.38230663846055501], [-0.38409317960223854, 0.39000000000000001, 0.35045469320011735], [-0.52740757430541685, 0.249, 0.29895529238584306], [0.34740441796104815, 0.246, 0.36414375737078841], [0.47786308046182885, 0.089999999999999997, 0.31637383012047438], [0.0047163311140149293, 0.23400000000000001, 0.49811846308566454], [-0.098930796747205507, 0.307, 0.46059660828759452], [-0.37158786444006753, 0.40500000000000003, 0.35509986295795459], [0.60261000309518942, 0.45800000000000002, 0.27338408212548115], [0.046168582994860159, 0.44600000000000001, 0.48158794144995187], [-0.049232649167827866, 0.28499999999999998, 0.48036694626611731], [1.0035582996961303, 0.050000000000000003, 0.15779578142846606], [0.52664582141189142, 0.218, 0.29921978282283035]] \ No newline at end of file From 2937c97fea8f2d55ba7787c979879ad8a363dd4b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 17:47:30 -0400 Subject: [PATCH 14/48] including correct fixtures --- src/py/crankshaft/test/fixtures/getis.json | 79 +++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json index 20fe1a9..71d7728 100644 --- a/src/py/crankshaft/test/fixtures/getis.json +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -1 +1,78 @@ -[[-0.4491160909028587, 0.43099999999999999, 0.32667395669425392], [-0.93788225530474956, 0.021999999999999999, 0.17415246162388542], [-0.4491160909028587, 0.44600000000000001, 0.32667395669425392], [-0.64751176059715954, 0.216, 0.25865039275726565], [-0.96347562762330119, 0.11700000000000001, 0.1676544447116961], [-0.20152426147617952, 0.049000000000000002, 0.42014433032225806], [-0.46619525535633877, 0.072999999999999995, 0.32053787260380839], [-0.74976349152754251, 0.16300000000000001, 0.22669858024734424], [-0.64751176059715954, 0.052999999999999999, 0.25865039275726565], [-0.56083197633324544, 0.156, 0.28745604294545346], [-0.24280805846929457, 0.29599999999999999, 0.40407704630238783], [-0.26928692847374969, 0.29099999999999998, 0.39385444517601476], [-0.67722330498207772, 0.44, 0.24913214171492104], [-0.011684382726338413, 0.16500000000000001, 0.49533871177388633], [-0.56473075182761978, 0.378, 0.28612845711175672], [-0.99072103490325258, 0.042999999999999997, 0.1609109084310375], [-0.28743876271061125, 0.45600000000000002, 0.38688819224593796], [0.13295980219309725, 0.156, 0.4471125856868039], [0.0063207047268819955, 0.19500000000000001, 0.49747842043259183], [-1.0415760797098768, 0.0070000000000000001, 0.1488041317793396], [-0.36942477246818262, 0.111, 0.35590556804466089], [0.17506551624725114, 0.22500000000000001, 0.43051407691606425], [-0.44363455341924385, 0.26900000000000002, 0.32865340985373048], [-0.43284736233536597, 0.33800000000000002, 0.33256283131987785], [-0.60291193010388022, 0.189, 0.27328363953070622], [0.035856268747985663, 0.20799999999999999, 0.48569848295465989], [0.089678570182411793, 0.123, 0.4642713229152271], [-1.1057224660076166, 0.0030000000000000001, 0.13442333147768415], [0.11677761660057258, 0.16800000000000001, 0.45351814086906583], [-0.013808110669475293, 0.29999999999999999, 0.49449153588626749], [-0.23290337417697893, 0.378, 0.40791821913691739], [0.22945579816510672, 0.41699999999999998, 0.40925733604961323], [6.5207630666532674, 0.002, 3.4975355944766306e-11], [-0.14672827921301743, 0.46200000000000002, 0.44167324829783083], [0.22908126100922532, 0.126, 0.40940287888500482], [-1.0840432095769956, 0.047, 0.13917282042962942], [4.0493273511601489, 0.017000000000000001, 2.5682527508119612e-05], [-0.93058949009641589, 0.067000000000000004, 0.17603297655968531], [-0.39105578866749485, 0.435, 0.34787799959221344], [6.4632623511010854, 0.0030000000000000001, 5.1234683162704187e-11], [0.29440403358766087, 0.098000000000000004, 0.38422459253128149], [-0.42044605311189681, 0.38400000000000001, 0.33707981556243372], [0.056785871984068648, 0.38700000000000001, 0.47735788415038005], [-0.15875022725747437, 0.36899999999999999, 0.43693283214176271], [0.14086541129997771, 0.20399999999999999, 0.44398813370614665], [6.0345506436163996, 0.014, 7.9702822119998018e-10], [-0.85883340027064359, 0.13400000000000001, 0.19521621870003536], [-0.93901580426004294, 0.14499999999999999, 0.17386131505073421], [2.052209368389633, 0.029000000000000001, 0.020074659704574338], [-0.2873860843780221, 0.45300000000000001, 0.38690835753455055], [0.11238736962962483, 0.27200000000000002, 0.45525813472736054], [-0.22667990193583551, 0.47599999999999998, 0.41033632813474463], [-0.66318887096788159, 0.060999999999999999, 0.25360479911357758], [-0.41451366242647014, 0.47599999999999998, 0.33924898169485895], [-0.65907426867863195, 0.14499999999999999, 0.25492403906163263], [-0.56550838435631712, 0.127, 0.28586401153505392], [-0.019571757025783815, 0.311, 0.49219249707263191], [-0.049735022936677371, 0.27700000000000002, 0.48016677335804769], [0.34564692985400752, 0.20899999999999999, 0.36480403618322277], [-0.055301366746933138, 0.26200000000000001, 0.47794918666392716], [-0.11227385176628404, 0.315, 0.45530313698210811], [-0.23091105262392728, 0.39900000000000002, 0.4086919520030482], [-0.05314541432928175, 0.38, 0.47880802357740249], [-0.048726126739086051, 0.29499999999999998, 0.48056877723722968], [-0.65725648028245498, 0.29999999999999999, 0.25550800720533962], [-0.29942829331962884, 0.35499999999999998, 0.38230663846055501], [-0.38409317960223854, 0.39000000000000001, 0.35045469320011735], [-0.52740757430541685, 0.249, 0.29895529238584306], [0.34740441796104815, 0.246, 0.36414375737078841], [0.47786308046182885, 0.089999999999999997, 0.31637383012047438], [0.0047163311140149293, 0.23400000000000001, 0.49811846308566454], [-0.098930796747205507, 0.307, 0.46059660828759452], [-0.37158786444006753, 0.40500000000000003, 0.35509986295795459], [0.60261000309518942, 0.45800000000000002, 0.27338408212548115], [0.046168582994860159, 0.44600000000000001, 0.48158794144995187], [-0.049232649167827866, 0.28499999999999998, 0.48036694626611731], [1.0035582996961303, 0.050000000000000003, 0.15779578142846606], [0.52664582141189142, 0.218, 0.29921978282283035]] \ No newline at end of file +[[-0.37378594228210338, 0.436, 0.35428178954558454], + [-0.95225687055925445, 0.065000000000000002, 0.17048336261993491], + [-0.69155363829052829, 0.313, 0.2446088425605718], + [-0.91005743729633215, 0.085999999999999993, 0.18139610975939735], + [-1.0132869736739361, 0.13200000000000001, 0.1554615538579156], + [-0.51711928006609076, 0.058999999999999997, 0.30253644777604194], + [-0.59915606178126646, 0.125, 0.27453440990643974], + [-0.65270328962244883, 0.314, 0.25697379065855031], + [-1.035818892716347, 0.016, 0.15014332094164029], + [-0.77022454873249124, 0.14899999999999999, 0.22058335212423197], + [-0.33280112478125556, 0.40500000000000003, 0.36964220456990138], + [-0.16619696958659569, 0.28000000000000003, 0.43400097210156352], + [-1.0189197578270577, 0.114, 0.15412053143437432], + [-0.58736356248618182, 0.378, 0.27847978008948393], + [-0.60815535737753856, 0.34699999999999998, 0.27154222002447759], + [-0.83654622542342216, 0.22, 0.20142384153812443], + [-0.27623528229314609, 0.48099999999999998, 0.39118367986278957], + [-0.17751012385821857, 0.40600000000000003, 0.42955385691066428], + [-0.42339868568527872, 0.499, 0.33600220182364238], + [-0.81090507212288232, 0.17000000000000001, 0.2087100936430748], + [-0.63864423309742102, 0.089999999999999997, 0.26152719947142589], + [-0.64036180904066409, 0.0070000000000000001, 0.2609687028452613], + [-0.62209633811403076, 0.217, 0.26693926135549806], + [2.6802178678493869, 0.019, 0.0036787128295844296], + [-0.53304848636954871, 0.314, 0.29700000849115427], + [-0.15325607655300719, 0.33900000000000002, 0.43909816861874729], + [-0.46599673142949988, 0.5, 0.32060892008190012], + [-0.67328453973424696, 0.34000000000000002, 0.25038314326507338], + [-0.65374072057172605, 0.27300000000000002, 0.25663943186717253], + [-0.042646376611931783, 0.33900000000000002, 0.48299171295314569], + [-0.50268629316736713, 0.36899999999999999, 0.30759242403716802], + [0.20061122903056294, 0.41199999999999998, 0.42050128853782709], + [4.7529210422438366, 0.002, 1.0024933114749501e-06], + [-0.17112791409785774, 0.45500000000000002, 0.43206159366732011], + [2.7269760223697093, 0.025000000000000001, 0.0031958841790815651], + [-1.2443986350856617, 0.023, 0.1066764425086173], + [2.6146417757460672, 0.058999999999999997, 0.0044660540776073621], + [-0.81089418923393874, 0.217, 0.20871321876749027], + [-0.96842037905996448, 0.051999999999999998, 0.16641723288102916], + [4.632217640632744, 0.014999999999999999, 1.8088477418132243e-06], + [1.7098997741760702, 0.066000000000000003, 0.043642204110986293], + [-0.68576671544237189, 0.30399999999999999, 0.24643011428027894], + [-0.10705643473845843, 0.46100000000000002, 0.45737210425657626], + [-0.54280332475246651, 0.29499999999999998, 0.29363261191793877], + [0.031036400804321389, 0.26000000000000001, 0.48762025500133288], + [4.5324686370772209, 0.021999999999999999, 2.9149167000142029e-06], + [-1.3114835366024926, 0.0089999999999999993, 0.09484722615634944], + [-1.3114835366024926, 0.0080000000000000002, 0.09484722615634944], + [4.6361327065826368, 0.001, 1.7749405375466765e-06], + [1.4367662910941781, 0.081000000000000003, 0.075392205783569644], + [2.8306701087322108, 0.031, 0.0023225301610166893], + [-0.26132309263094761, 0.47599999999999998, 0.39692167988511706], + [-0.78055710491931951, 0.13400000000000001, 0.21753151443265129], + [-0.64234747392787916, 0.33300000000000002, 0.26032379813179585], + [-1.3114835366024926, 0.002, 0.09484722615634944], + [-0.57318180672306673, 0.26400000000000001, 0.28326080073223758], + [1.4602423896649199, 0.085999999999999993, 0.072111734510341252], + [-0.10852292180607989, 0.33700000000000002, 0.45679044945885094], + [0.25007822707788863, 0.25800000000000001, 0.4012634266959908], + [-0.15535540716918589, 0.36799999999999999, 0.43827056859443592], + [-0.15387021291338146, 0.34300000000000003, 0.43885603550771368], + [-0.23212287899595097, 0.434, 0.40822128763053078], + [-0.34331698296028734, 0.255, 0.36568000291448455], + [-0.21917003612352226, 0.38300000000000001, 0.41325879787572739], + [-0.79956517297385543, 0.20000000000000001, 0.21198138620767248], + [-0.57435424239923683, 0.20100000000000001, 0.28286405516217616], + [0.17090357402483336, 0.17599999999999999, 0.43214979317279412], + [-0.45071423902325392, 0.39500000000000002, 0.32609775926146312], + [0.76813695849036678, 0.16500000000000001, 0.22120291089357114], + [0.43164294033005113, 0.14899999999999999, 0.33300047214912831], + [0.3756136313798501, 0.16, 0.35360207763890095], + [-0.35027449210479344, 0.47599999999999998, 0.36306635326063197], + [-0.48157903675663827, 0.45500000000000002, 0.31505251051764338], + [0.82850242294957743, 0.187, 0.20369301166122333], + [0.32654794875356141, 0.23599999999999999, 0.3720049088953894], + [-0.30807434754194035, 0.497, 0.37901288012280243], + [0.50368346846574807, 0.159, 0.30724191359295261], + [0.75454835573966283, 0.13500000000000001, 0.22526001043744448]] From 5d109acd8df8e481ce136b1a001ed7fea605b61e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 13 Sep 2016 17:59:15 -0400 Subject: [PATCH 15/48] remove debug messages --- src/py/crankshaft/test/test_clustering_getis.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index fc26fc7..c10c8ab 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -19,20 +19,19 @@ class GetisTest(unittest.TestCase): def setUp(self): plpy._reset() - print(help(cc)) self.neighbors_data = json.loads(open(fixture_file('neighbors_getis.json')).read()) self.getis_data = json.loads(open(fixture_file('getis.json')).read()) def test_getis_ord(self): """Test Getis-Ord's G*""" data = [ { 'id': d['id'], - 'attr1': d['hr8893'], + 'value': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.getis_ord('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = cc.getis_ord('subquery', 'value', 'knn', 5, 999, 'the_geom', 'id') result = [(row[0], row[1]) for row in result] - expected = self.getis_data + expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): - self.assertAlmostEqual(res_val, exp_val) - self.assertEqual(res_quad, exp_quad) + self.assertAlmostEqual(res_z, exp_z, delta=1e-2) + self.assertEqual(res_p, exp_p, delta=1e-2) From b5445da3030ddfc4d7f93e6d08544d3247a7d6f5 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 14 Sep 2016 12:45:43 +0000 Subject: [PATCH 16/48] remove kinks in test --- src/py/crankshaft/test/test_clustering_getis.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index fc26fc7..0f3eb3f 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -19,20 +19,19 @@ class GetisTest(unittest.TestCase): def setUp(self): plpy._reset() - print(help(cc)) self.neighbors_data = json.loads(open(fixture_file('neighbors_getis.json')).read()) self.getis_data = json.loads(open(fixture_file('getis.json')).read()) def test_getis_ord(self): """Test Getis-Ord's G*""" data = [ { 'id': d['id'], - 'attr1': d['hr8893'], + 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.getis_ord('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = cc.getis_ord('subquery', 'value', 'knn', 5, 999, 'the_geom', 'cartodb_id') result = [(row[0], row[1]) for row in result] - expected = self.getis_data + expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): - self.assertAlmostEqual(res_val, exp_val) - self.assertEqual(res_quad, exp_quad) + self.assertAlmostEqual(res_z, exp_z, delta=1e-2) + self.assertAlmostEqual(res_p, exp_p, delta=1e-2) From 11176b71b319e00a3ae177cd133e38e3207bbcdb Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 19 Sep 2016 10:47:59 -0400 Subject: [PATCH 17/48] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9bb2e75..3de84c4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,6 +2,8 @@ - [ ] All declared geometries are `geometry(Geometry, 4326)` for general geoms, or `geometry(Point, 4326)` - [ ] Existing functions in crankshaft python library called from the extension are kept at least from version N to version N+1 (to avoid breakage during upgrades). - [ ] Docs for public-facing functions are written -- [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`. -- [ ] If appropriate, new functions accepts an arbitrary query as an input (see [Crankshaft Issue #6](https://github.com/CartoDB/crankshaft/issues/6) for more information) - +- [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore +- [ ] Video explaining the analysis and showing examples +- [ ] Analysis Documentation written [template](https://docs.google.com/a/cartodb.com/document/d/1X2KOtaiEBKWNMp8UjwcLB-kE9aIOw09aOjX3oaCjeME/edit?usp=sharing) +- [ ] Smoke test written + From 06f0cb0dc43f8366f65d1f9db99544c416fc587f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 15:45:10 +0000 Subject: [PATCH 18/48] updating how p values are tested --- src/py/crankshaft/test/test_clustering_getis.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 0f3eb3f..2fe0c54 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -15,8 +15,11 @@ from crankshaft import random_seeds import json class GetisTest(unittest.TestCase): - """Testing class for Getis-Ord's G funtion""" - + """Testing class for Getis-Ord's G funtion + This test replicates the work done in PySAL documentation: + https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/autocorrelation.html#local-g-and-g + """ + def setUp(self): plpy._reset() self.neighbors_data = json.loads(open(fixture_file('neighbors_getis.json')).read()) @@ -34,4 +37,6 @@ class GetisTest(unittest.TestCase): expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): self.assertAlmostEqual(res_z, exp_z, delta=1e-2) - self.assertAlmostEqual(res_p, exp_p, delta=1e-2) + if exp_p <= 0.05: + self.assertTrue(res_p < 0.05) + From 2ede55d1658e3a499990eb7a5c7b88a69b480c97 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 12:17:01 -0400 Subject: [PATCH 19/48] pep8 updates --- src/py/crankshaft/crankshaft/clustering/getis.py | 9 +++++---- src/py/crankshaft/test/test_clustering_getis.py | 16 +++++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index 1ee425a..a593e64 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -11,12 +11,13 @@ import crankshaft.pysal_utils as pu # High level interface --------------------------------------- + def getis_ord(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col): """ Getis-Ord's G* - Implementation building neighbors with a PostGIS database and PySAL's Getis-Ord's G* - hotspot/coldspot module. + Implementation building neighbors with a PostGIS database and PySAL's + Getis-Ord's G* hotspot/coldspot module. Andy Eschbacher """ @@ -41,11 +42,11 @@ def getis_ord(subquery, attr, attr_vals = pu.get_attributes(result) - ## build PySAL weight object + # build PySAL weight object weight = pu.get_weight(result, w_type, num_ngbrs) # calculate Getis-Ord's G* z- and p-values getis = ps.esda.getisord.G_Local(attr_vals, weight, - star=True, permutations=permutations) + star=True, permutations=permutations) return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order) diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 2fe0c54..f56d5fb 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -14,29 +14,31 @@ import crankshaft.pysal_utils as pu from crankshaft import random_seeds import json + class GetisTest(unittest.TestCase): """Testing class for Getis-Ord's G funtion This test replicates the work done in PySAL documentation: https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/autocorrelation.html#local-g-and-g """ - + def setUp(self): plpy._reset() - self.neighbors_data = json.loads(open(fixture_file('neighbors_getis.json')).read()) + self.neighbors_data = json.loads( + open(fixture_file('neighbors_getis.json')).read()) self.getis_data = json.loads(open(fixture_file('getis.json')).read()) def test_getis_ord(self): """Test Getis-Ord's G*""" - data = [ { 'id': d['id'], - 'attr1': d['value'], - 'neighbors': d['neighbors'] } for d in self.neighbors_data] + data = [{'id': d['id'], + 'attr1': d['value'], + 'neighbors': d['neighbors']} for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.getis_ord('subquery', 'value', 'knn', 5, 999, 'the_geom', 'cartodb_id') + result = cc.getis_ord('subquery', 'value', + 'knn', 5, 999, 'the_geom', 'cartodb_id') result = [(row[0], row[1]) for row in result] expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): self.assertAlmostEqual(res_z, exp_z, delta=1e-2) if exp_p <= 0.05: self.assertTrue(res_p < 0.05) - From ee4eb795b7e765262dc1a4448529c43cdea4f8c9 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 19:24:23 +0000 Subject: [PATCH 20/48] adding getis fixture file --- src/pg/test/fixtures/getis_data.sql | 103 ++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/pg/test/fixtures/getis_data.sql diff --git a/src/pg/test/fixtures/getis_data.sql b/src/pg/test/fixtures/getis_data.sql new file mode 100644 index 0000000..e7b2f69 --- /dev/null +++ b/src/pg/test/fixtures/getis_data.sql @@ -0,0 +1,103 @@ +SET client_min_messages TO WARNING; +\set ECHO none + +-- +-- Getis-Ord's G* test dataset, subsetted from PySAL examples: +-- https://github.com/pysal/pysal/tree/952ea04029165048a774d9a1846cf86ad000c096/pysal/examples/stl +-- + + +CREATE TABLE getis_data ( + cartodb_id integer, + the_geom geometry(Geometry,4326), + hr8893 numeric +); + + +ALTER TABLE getis_data OWNER TO contrib_regression; + + +COPY getis_data (cartodb_id, the_geom, hr8893) FROM stdin; +22 0106000020E61000000100000001030000000100000007000000000000E0B10056C0000000C0B8964340FFFFFFFF4C1756C00000002054964340000000A00F1E56C00000004072964340000000C02D1E56C0000000A0439B434000000060381E56C00000000036B04340000000E0E20056C0000000608CB04340000000E0B10056C0000000C0B8964340 10.8557430000000004 +32 0106000020E6100000010000000103000000010000000B000000FFFFFF1FC26656C0FFFFFFBFE25E4340000000A0D86656C0000000E0976F4340000000A03A6956C0000000C0966F434000000020526956C0000000E08A7F4340000000E0F26556C000000000C87F4340000000E0066656C0000000209C834340000000407F5056C0000000803C83434000000020635056C0000000E016814340000000A0F45056C0000000A0F980434000000060D25056C000000060FA5E4340FFFFFF1FC26656C0FFFFFFBFE25E4340 9.92424500000000087 +10 0106000020E610000001000000010300000001000000170000000000000002CD56C000000080CDCC4340000000A054D456C000000020CCD74340000000607ED756C000000000C1DC434000000020E6D756C00000006071E143400000004007BB56C00000000007E2434000000080FABA56C0000000A079EC43400000000040B856C0000000E0D6EB4340FFFFFF3FEEA456C0000000A037EC4340000000C0A9A556C0000000A0ADE7434000000040F3A656C0000000E09FE543400000004063A956C0000000E034DA4340FFFFFF9F04A956C00000008005D74340000000402FA756C00000008069D243400000004046A556C00000002068C84340000000009EA556C0FFFFFF7F3CC34340000000C0C3A756C000000080BCB543400000006082A756C00000004051B2434000000040AABC56C00000006046B343400000006053C256C0FFFFFF7FE2B84340000000E01EC456C000000080ABBC4340000000C0FDC556C0000000E0B3C3434000000000FFC956C000000060BBC643400000000002CD56C000000080CDCC4340 3.79060700000000006 +43 0106000020E6100000010000000103000000010000000F0000000000004025D856C000000020FA1A43400000008092E256C000000060481B434000000060BCE256C0000000C023144340000000A0D7E856C0FFFFFF1F1B14434000000020BEE956C0000000C030144340FFFFFF9FB0E956C0FFFFFF1F1425434000000000D4E956C0000000C00D5A4340000000A0D3E956C0000000202C5A43400000000004E656C00000004066574340000000E0EEE356C0000000A0E35643400000008099DF56C0000000601B5A43400000000033DB56C0000000804A5B43400000004001D856C00000006079594340000000E0A7D756C0000000E0553543400000004025D856C000000020FA1A4340 5.93309800000000021 +6 0106000020E6100000010000000103000000010000000F000000000000A00F4256C0000000E008D4434000000000674956C00000004015D44340000000608C4956C00000004098E64340FFFFFFBF434C56C0FFFFFF3F77E84340000000004C4E56C000000020E5E74340000000C0624E56C0FFFFFF3F97F5434000000020B44956C000000000AFF54340000000E0C64956C00000004009074440FFFFFF1F523056C0FFFFFFDF91074440000000C0EB2F56C000000040BBE54340000000E0B93056C0000000E09FE54340000000E0D63056C0000000007DDE4340000000E0213456C0000000005ADE4340000000802E3456C000000020F7D34340000000A00F4256C0000000E008D44340 9.04867300000000085 +16 0106000020E6100000010000000103000000010000001500000000000020D73356C000000060729B4340000000201F4956C000000000BE9B4340000000A0E34856C000000060CCAC434000000040094256C0FFFFFFDFB1AC4340000000A00F4256C0000000E008D44340000000802E3456C000000020F7D34340000000E0083456C0000000E0ADCA4340000000801D2E56C0000000A06FCA4340FFFFFF7F132E56C00000000079C34340000000607F2956C0000000402EC3434000000080652956C0000000A0EAC04340FFFFFF5FF22756C000000060E5C0434000000080F52756C000000080DDBE434000000020B32656C0000000E0E7BE434000000000AC2656C0FFFFFFFF38BD4340FFFFFFDFC12556C0000000C026BD434000000060C72556C000000060BAB9434000000040441E56C000000020C9B9434000000060381E56C00000000036B04340000000C02D1E56C0000000A0439B434000000020D73356C000000060729B4340 0.74509000000000003 +29 0106000020E610000001000000010300000001000000080000000000002025FD55C0FFFFFF1F7F6D434000000080C61056C0000000A04C6D4340000000A0631756C000000000D56D4340000000C05E1756C0FFFFFF5F24754340FFFFFFFF4C1756C00000002054964340000000E0B10056C0000000C0B89643400000006029FD55C000000080C09643400000002025FD55C0FFFFFF1F7F6D4340 3.12759000000000009 +54 0106000020E6100000010000000103000000010000000F000000FFFFFF1F090C57C0000000202E024340000000E0AA0B57C00000004000154340000000A0C10C57C0000000802015434000000060850C57C000000080EC244340FFFFFF9FB0E956C0FFFFFF1F1425434000000020BEE956C0000000C030144340000000A0D7E856C0FFFFFF1F1B144340FFFFFF3FF2E856C0000000A0A3064340FFFFFFDFD2F956C000000040E2064340000000C01EFB56C0000000201406434000000060BBFB56C0000000809A044340000000605AFD56C0000000E059054340000000405FFE56C0000000E06B014340000000A0F60157C0000000C081014340FFFFFF1F090C57C0000000202E024340 2.06432400000000005 +13 0106000020E6100000010000000103000000010000000F000000000000009EA556C0FFFFFF7F3CC343400000004046A556C00000002068C84340000000402FA756C00000008069D24340FFFFFF9F04A956C00000008005D743400000004063A956C0000000E034DA434000000040F3A656C0000000E09FE54340000000E0F09E56C0000000005AE54340FFFFFF3F069F56C000000040B5E04340FFFFFF1F0E9856C000000020A3E0434000000060109856C0000000E07ED5434000000040E29556C0000000A077D54340FFFFFFDFDC9556C0000000A0EDD1434000000080399356C0000000A0D8D1434000000020699356C0000000E029C34340000000009EA556C0FFFFFF7F3CC34340 0 +28 0106000020E61000000100000001030000000100000008000000000000C05E1756C0FFFFFF5F24754340000000C0DD2C56C0000000407A75434000000080DE3356C0000000406375434000000020D73356C000000060729B4340000000C02D1E56C0000000A0439B4340000000A00F1E56C00000004072964340FFFFFFFF4C1756C00000002054964340000000C05E1756C0FFFFFF5F24754340 1.57115800000000005 +36 0106000020E6100000010000000103000000010000000D00000000000000EE2C56C000000060424E434000000040F72C56C000000000486A4340000000C0DD2C56C0000000407A754340000000C05E1756C0FFFFFF5F24754340000000A0631756C000000000D56D434000000080C61056C0000000A04C6D4340000000C0BE1056C0000000A0065F4340000000407C1256C0FFFFFF7FA75E434000000000BD1156C000000020A954434000000040D01256C0000000605C524340000000404F1256C000000040734F434000000040011156C000000000AF4D434000000000EE2C56C000000060424E4340 0 +68 0106000020E61000000100000001030000000100000006000000000000809F2D56C00000002078CD4240000000E0F64256C0FFFFFFDF38CD424000000000CE4956C00000004053CD4240000000C0E94956C000000080CDEE424000000020682D56C0FFFFFF7F00EF4240000000809F2D56C00000002078CD4240 3.1461920000000001 +27 0106000020E6100000010000000103000000010000000D000000000000407F5056C0000000803C83434000000060615056C000000040A99B4340000000201F4956C000000000BE9B434000000020D73356C000000060729B434000000080DE3356C00000004063754340000000C0DD2C56C0000000407A75434000000040F72C56C000000000486A4340000000202B4956C000000080AC69434000000040454956C000000040E25E434000000060D25056C000000060FA5E4340000000A0F45056C0000000A0F980434000000020635056C0000000E016814340000000407F5056C0000000803C834340 1.5920399999999999 +40 0106000020E6100000010000000103000000010000000F00000000000000B89056C0000000C03144434000000000419456C0000000A03D4A434000000020609356C000000020CC55434000000020578F56C0000000A0725D4340000000004D8C56C0000000C05E614340000000E0F48A56C000000020B164434000000060CA8756C00000002077664340000000C0A68856C0000000C08A64434000000020758A56C0000000E0F4624340000000A0948C56C0000000E0AA5C4340FFFFFF7FF18C56C000000080A5594340000000C0BF8B56C0000000A052544340000000E0C18B56C0000000601D4E4340000000A06B8F56C0000000000A48434000000000B89056C0000000C031444340 45.9054059999999993 +52 0106000020E61000000100000001030000000100000008000000000000E0562D56C0000000208910434000000020864856C00000000054104340FFFFFF9FB84956C0000000409110434000000080A84956C0000000A05B1B4340000000803D4956C0FFFFFF9FB63C434000000080062D56C0FFFFFF7FAE3C434000000060492D56C0000000603E214340000000E0562D56C00000002089104340 4.94153299999999973 +59 0106000020E6100000010000000103000000010000000F0000000000006010C756C000000020A7DE4240FFFFFF9FF7C956C0FFFFFF7F75DE424000000060FBC956C0000000A01ED94240000000C080CA56C0000000001BD94240000000C086D456C000000040CED942400000000065D456C00000004041E442400000008044E256C0000000E0BEE44240FFFFFFFF04E256C0FFFFFFBFD013434000000060BCE256C0000000C0231443400000008092E256C000000060481B43400000004025D856C000000020FA1A4340FFFFFF5F2CD856C0000000E0381A43400000000009D656C0FFFFFFDF1C1A4340000000A08DC656C000000000241A43400000006010C756C000000020A7DE4240 6.82879400000000025 +1 0106000020E61000000100000001030000000100000010000000FFFFFF3F746556C00000002049FD434000000080316556C0000000A0240C4440000000C0A76656C000000000330C444000000000C76656C0000000606C11444000000000106756C000000000A0294440000000C0375156C0000000402F2A444000000020305156C0000000E08D244440000000E0E54956C00000004094244440000000E0C64956C0000000400907444000000020B44956C000000000AFF54340000000C0624E56C0FFFFFF3F97F54340000000E05B5A56C00000006082F5434000000060655A56C00000002040F7434000000060B15F56C0000000807CF74340000000008A5F56C0000000C076FD4340FFFFFF3F746556C00000002049FD4340 1.62445799999999996 +2 0106000020E6100000010000000103000000010000001000000000000080FABA56C0000000A079EC43400000004007BB56C00000000007E2434000000020E6D756C00000006071E14340000000006ED856C000000000E2E64340FFFFFF7FBFDC56C00000006078EE434000000000DDDC56C0000000A04FF1434000000080C7DB56C0000000206FF34340000000808BDB56C0000000C0FEF54340000000A09FDC56C0000000A018F94340000000C02FDF56C000000080BC004440000000A041E056C0000000008A084440FFFFFF3F08E156C0000000C038114440000000406BE056C0000000A0A8194440FFFFFFBFF5BA56C0000000E02A1944400000004014BB56C000000060E70D444000000080FABA56C0000000A079EC4340 2.25549199999999983 +3 0106000020E61000000100000001030000000100000018000000000000A0D87F56C000000080FEF3434000000020048056C0FFFFFF7FB00E444000000080CB7D56C0000000E0F411444000000080237B56C0000000800812444000000060227956C0000000400F114440000000A04A7856C00000002084124440000000C0667756C0000000E0B810444000000080187656C0000000A001124440000000A0037556C0FFFFFF5F36104440000000005C7356C0000000206510444000000080D27156C00000004081114440000000A0657056C000000040EE104440000000A0EF6D56C000000060AE12444000000000BE6C56C0000000E073124440000000802A6A56C0000000402115444000000000C76656C0000000606C114440000000C0A76656C000000000330C444000000080316556C0000000A0240C4440FFFFFF3F746556C00000002049FD4340000000602C6D56C00000004009FD4340FFFFFF5F786D56C0FFFFFFDFC7F54340000000E0447156C000000080A6F54340000000A09C7156C0FFFFFF9F41F44340000000A0D87F56C000000080FEF34340 1.46788999999999992 +4 0106000020E610000001000000010300000001000000160000000000008076A556C000000020BCF0434000000060BCA256C0FFFFFF5F7BF5434000000060F5A056C000000040AEFE4340000000204C9C56C00000008095024440000000A0609B56C000000060EC084440FFFFFFFFC49956C0000000C00C0A444000000000E19856C000000040490F444000000040A79656C0000000805C10444000000080199456C0000000E0120E4440FFFFFFBF879256C00000006094084440000000A0699156C00000006012084440000000807E9056C000000000F408444000000060378F56C00000000049074440000000A0E98C56C0FFFFFF5FE4074440000000E0F48B56C0000000201009444000000020618856C000000040ED08444000000020A88556C000000060CF0A4440FFFFFFDF0C8456C000000080250D444000000020048056C0FFFFFF7FB00E4440000000A0D87F56C000000080FEF34340FFFFFF3FDF7F56C0000000C047F043400000008076A556C000000020BCF04340 2.4842559999999998 +5 0106000020E61000000100000001030000000100000012000000FFFFFF3FEEA456C0000000A037EC43400000000040B856C0000000E0D6EB434000000080FABA56C0000000A079EC43400000004014BB56C000000060E70D444000000020B9AC56C0000000409F0D4440000000A094AD56C0000000005B0A4440000000A07FAB56C0000000A0C1094440FFFFFFBFA8AB56C000000060DD054440000000E04BA956C0FFFFFFFFB30344400000002042A756C0FFFFFFBF900344400000002005A756C0000000409502444000000060D2A756C000000040EE014440000000C0D9A656C0000000008E004440000000E0C0A656C0000000E0F1FD434000000060F5A056C000000040AEFE434000000060BCA256C0FFFFFF5F7BF543400000008076A556C000000020BCF04340FFFFFF3FEEA456C0000000A037EC4340 0 +15 0106000020E6100000010000000103000000010000000B000000000000806CDC56C00000006020A9434000000080A4ED56C000000020E3A94340000000A050EE56C0FFFFFFDF8BAB434000000000C4ED56C00000000058CD4340000000A039EE56C0000000C062CD4340000000C041EE56C00000006068D44340000000E031EE56C000000080D5D74340000000A054D456C000000020CCD743400000000002CD56C000000080CDCC434000000040CDDD56C0000000407EBA4340000000806CDC56C00000006020A94340 1.9498120000000001 +7 0106000020E6100000010000000103000000010000001C000000000000C00D6D56C0000000A096C34340000000E0847B56C00000000075C3434000000060747B56C00000002076C74340000000601B7F56C0000000203CDA4340FFFFFF3FDF7F56C0000000C047F04340000000A0D87F56C000000080FEF34340000000A09C7156C0FFFFFF9F41F44340000000E0447156C000000080A6F54340FFFFFF5F786D56C0FFFFFFDFC7F54340000000602C6D56C00000004009FD4340FFFFFF3F746556C00000002049FD4340000000008A5F56C0000000C076FD434000000060B15F56C0000000807CF7434000000060655A56C00000002040F74340000000E05B5A56C00000006082F54340000000C0624E56C0FFFFFF3F97F54340000000004C4E56C000000020E5E7434000000080B55056C0000000C010E94340FFFFFFDF215256C0000000C096E5434000000020475556C0000000A0D3E14340000000A0245A56C0000000200DDF434000000060E75B56C000000000C0DF434000000000F25B56C0000000C0DBD74340FFFFFF5F635F56C0FFFFFFFF91D74340000000E04F5F56C000000020C1D24340FFFFFF7F996256C00000004094D24340000000A08E6256C0000000C0AAC34340000000C00D6D56C0000000A096C34340 6.02948899999999988 +8 0106000020E6100000010000000103000000010000000E000000FFFFFF1F72F656C000000080A1D443400000004048F656C0000000008EF94340000000A09FDC56C0000000A018F94340000000808BDB56C0000000C0FEF5434000000080C7DB56C0000000206FF3434000000000DDDC56C0000000A04FF14340FFFFFF7FBFDC56C00000006078EE4340000000006ED856C000000000E2E6434000000020E6D756C00000006071E14340000000607ED756C000000000C1DC4340000000A054D456C000000020CCD74340000000E031EE56C000000080D5D74340000000C041EE56C00000006068D44340FFFFFF1F72F656C000000080A1D44340 1.8003849999999999 +9 0106000020E6100000010000000103000000010000001200000000000040E88956C00000004046C3434000000020699356C0000000E029C3434000000080399356C0000000A0D8D14340FFFFFFDFDC9556C0000000A0EDD1434000000040E29556C0000000A077D5434000000060109856C0000000E07ED54340FFFFFF1F0E9856C000000020A3E04340FFFFFF3F069F56C000000040B5E04340000000E0F09E56C0000000005AE5434000000040F3A656C0000000E09FE54340000000C0A9A556C0000000A0ADE74340FFFFFF3FEEA456C0000000A037EC43400000008076A556C000000020BCF04340FFFFFF3FDF7F56C0000000C047F04340000000601B7F56C0000000203CDA434000000060747B56C00000002076C74340000000E0847B56C00000000075C3434000000040E88956C00000004046C34340 4.58125099999999996 +30 0106000020E6100000010000000103000000010000000D0000000000000033DB56C0000000804A5B43400000008099DF56C0000000601B5A4340000000E0EEE356C0000000A0E35643400000000004E656C00000004066574340000000A0D3E956C0000000202C5A4340000000A02EE956C0FFFFFF1F09884340000000A0D9E856C0FFFFFFDF03934340000000A0B8DA56C000000060E592434000000080F4D056C0000000605F9243400000000038D156C0000000A0817F434000000000A0D156C0000000A0FF6B4340000000E0C8DA56C000000080986C43400000000033DB56C0000000804A5B4340 4.41689600000000038 +11 0106000020E61000000100000001030000000100000013000000000000A08E6256C0000000C0AAC34340FFFFFF7F996256C00000004094D24340000000E04F5F56C000000020C1D24340FFFFFF5F635F56C0FFFFFFFF91D7434000000000F25B56C0000000C0DBD7434000000060E75B56C000000000C0DF4340000000A0245A56C0000000200DDF434000000020475556C0000000A0D3E14340FFFFFFDF215256C0000000C096E5434000000080B55056C0000000C010E94340000000004C4E56C000000020E5E74340FFFFFFBF434C56C0FFFFFF3F77E84340000000608C4956C00000004098E6434000000000674956C00000004015D44340000000A00F4256C0000000E008D4434000000040094256C0FFFFFFDFB1AC4340000000A0E34856C000000060CCAC434000000020666256C0FFFFFF9FBFAC4340000000A08E6256C0000000C0AAC34340 1.44743599999999994 +12 0106000020E6100000010000000103000000010000001400000000000040441E56C000000020C9B9434000000060C72556C000000060BAB94340FFFFFFDFC12556C0000000C026BD434000000000AC2656C0FFFFFFFF38BD434000000020B32656C0000000E0E7BE434000000080F52756C000000080DDBE4340FFFFFF5FF22756C000000060E5C0434000000080652956C0000000A0EAC04340000000607F2956C0000000402EC34340FFFFFF7F132E56C00000000079C34340000000801D2E56C0000000A06FCA4340000000E0083456C0000000E0ADCA4340000000802E3456C000000020F7D34340000000E0213456C0000000005ADE4340000000E0D63056C0000000007DDE4340000000E0B93056C0000000E09FE54340000000C0EB2F56C000000040BBE54340FFFFFF5F741E56C00000008040E54340000000605A1E56C0000000E042D3434000000040441E56C000000020C9B94340 1.19196600000000008 +14 0106000020E6100000010000000103000000010000000A000000000000E0E20056C0000000608CB0434000000060381E56C00000000036B0434000000040441E56C000000020C9B94340000000605A1E56C0000000E042D3434000000040BB0356C00000000030D4434000000060CF0356C000000080C9D74340FFFFFFBFFDFD55C0000000E02BD84340000000A0C5FD55C0000000400DBE434000000040E60056C0FFFFFF9F20BE4340000000E0E20056C0000000608CB04340 1.60801700000000003 +17 0106000020E61000000100000001030000000100000011000000000000A0B8DA56C000000060E5924340000000806CDC56C00000006020A9434000000040CDDD56C0000000407EBA43400000000002CD56C000000080CDCC434000000000FFC956C000000060BBC64340000000C0FDC556C0000000E0B3C34340000000E01EC456C000000080ABBC43400000006053C256C0FFFFFF7FE2B8434000000040AABC56C00000006046B34340000000806EB656C0000000A0DBAC4340000000C0E0B156C0000000A0FDA54340000000C03CAF56C000000040B89F4340000000A0DEAE56C0FFFFFF7FC49C4340000000E0F3CB56C0FFFFFF9F039D4340000000205CCC56C0000000805392434000000080F4D056C0000000605F924340000000A0B8DA56C000000060E5924340 4.17331800000000008 +18 0106000020E6100000010000000103000000010000000E000000FFFFFF3FF56C56C000000020977F4340000000A07A6D56C0000000605DAD4340000000800F6D56C0000000A06CAD4340000000C00D6D56C0000000A096C34340000000A08E6256C0000000C0AAC3434000000020666256C0FFFFFF9FBFAC4340000000A0E34856C000000060CCAC4340000000201F4956C000000000BE9B434000000060615056C000000040A99B4340000000407F5056C0000000803C834340000000E0066656C0000000209C834340000000E0F26556C000000000C87F434000000020526956C0000000E08A7F4340FFFFFF3FF56C56C000000020977F4340 3.78325200000000006 +19 0106000020E61000000100000001030000000100000009000000000000A0B78956C0000000A0BD7F4340000000C0C08956C0000000000BA1434000000040E88956C00000004046C34340000000E0847B56C00000000075C34340000000C00D6D56C0000000A096C34340000000800F6D56C0000000A06CAD4340000000A07A6D56C0000000605DAD4340FFFFFF3FF56C56C000000020977F4340000000A0B78956C0000000A0BD7F4340 2.08513599999999988 +20 0106000020E61000000100000001030000000100000015000000000000C0C08956C0000000000BA1434000000060188D56C00000008040A04340000000C0278D56C0000000E0E29C4340000000E04C9456C000000080CC9C4340000000A0579456C0000000E0B4964340FFFFFFFF819F56C0000000806D9643400000006068A056C0FFFFFF1FBB944340000000203AA156C0000000E0CA9743400000002061A456C000000080AE974340000000C08CA556C0000000A0BB964340FFFFFF9F44A556C0000000609794434000000040EFA656C0000000C00C8F43400000004050A756C0FFFFFFBFEA9343400000000062A656C0FFFFFFFF6A9B434000000080DCA756C0000000208AAE43400000006082A756C00000004051B24340000000C0C3A756C000000080BCB54340000000009EA556C0FFFFFF7F3CC3434000000020699356C0000000E029C3434000000040E88956C00000004046C34340000000C0C08956C0000000000BA14340 2.17630200000000018 +21 0106000020E6100000010000000103000000010000001B00000000000040EFA656C0000000C00C8F4340FFFFFF5F1DA756C000000020C58D434000000020E1A456C0000000E00F84434000000020D1A456C0000000A0B2804340FFFFFFDF3AA356C000000040247C4340000000800BA256C0FFFFFF5F987A434000000020409F56C000000000CD7B4340000000E0119E56C000000060C67A434000000080F2A156C000000040207243400000004080A456C0000000A0876F43400000004024A856C0000000E0BD704340FFFFFFDFCEAA56C000000060B67743400000004030AD56C000000060D6844340FFFFFF1F49AD56C00000006072874340000000802FAC56C000000060FE8B434000000000DFAD56C00000008075924340FFFFFFDFF6AD56C00000006012994340000000A0DEAE56C0FFFFFF7FC49C4340000000C03CAF56C000000040B89F4340000000C0E0B156C0000000A0FDA54340000000806EB656C0000000A0DBAC434000000040AABC56C00000006046B343400000006082A756C00000004051B2434000000080DCA756C0000000208AAE43400000000062A656C0FFFFFFFF6A9B43400000004050A756C0FFFFFFBFEA93434000000040EFA656C0000000C00C8F4340 6.30934699999999982 +23 0106000020E6100000010000000103000000010000000B000000FFFFFF1F491457C000000060E99F4340FFFFFF5F1C1457C00000000075AC4340000000A050EE56C0FFFFFFDF8BAB434000000080A4ED56C000000020E3A94340000000806CDC56C00000006020A94340000000A0B8DA56C000000060E5924340000000A0D9E856C0FFFFFFDF03934340000000A02EE956C0FFFFFF1F0988434000000020290757C0FFFFFF3F92884340000000A0B00657C000000080FE9E4340FFFFFF1F491457C000000060E99F4340 4.21135400000000004 +24 0106000020E61000000100000001030000000100000019000000000000A0B78956C0000000A0BD7F4340FFFFFF1FDD9156C000000060B47F434000000000DA9156C0000000205D764340000000A0769456C00000006063764340000000C06F9A56C0000000A02D7B4340000000E0119E56C000000060C67A434000000020409F56C000000000CD7B4340000000800BA256C0FFFFFF5F987A4340FFFFFFDF3AA356C000000040247C434000000020D1A456C0000000A0B280434000000020E1A456C0000000E00F844340FFFFFF5F1DA756C000000020C58D434000000040EFA656C0000000C00C8F4340FFFFFF9F44A556C00000006097944340000000C08CA556C0000000A0BB9643400000002061A456C000000080AE974340000000203AA156C0000000E0CA9743400000006068A056C0FFFFFF1FBB944340FFFFFFFF819F56C0000000806D964340000000A0579456C0000000E0B4964340000000E04C9456C000000080CC9C4340000000C0278D56C0000000E0E29C434000000060188D56C00000008040A04340000000C0C08956C0000000000BA14340000000A0B78956C0000000A0BD7F4340 0.804810000000000025 +47 0106000020E61000000100000001030000000100000009000000FFFFFF9FD60956C0000000003521434000000020F21756C000000040DA20434000000060492D56C0000000603E21434000000080062D56C0FFFFFF7FAE3C434000000000EE2C56C000000060424E434000000040011156C000000000AF4D434000000040780956C0000000005A4D4340000000808A0956C000000080C1494340FFFFFF9FD60956C00000000035214340 0.969790999999999959 +25 0106000020E6100000010000000103000000010000001E000000000000A0491957C0FFFFFF9F355E434000000020381A57C0000000A05B61434000000000201957C0000000803D654340000000802D1957C000000040E867434000000060B41B57C0000000C078694340000000A0611E57C000000040A06E4340000000C0FF1F57C00000006083754340000000803C2457C0000000E0ED7B4340000000A0332457C00000004047804340FFFFFF9FB41B57C0FFFFFF5F08A04340FFFFFF1F491457C000000060E99F4340000000A0B00657C000000080FE9E434000000020290757C0FFFFFF3F92884340FFFFFFBFC70857C000000020EC874340000000C0F20957C000000020DE764340000000C0E50A57C000000040E172434000000000260A57C0000000A03A71434000000000750A57C000000080796F4340FFFFFF7F310957C00000000064684340FFFFFF3FFF0B57C000000020765E434000000000060E57C000000000B45B434000000000640E57C00000000022594340000000A0C40C57C0000000801D57434000000040600C57C0000000C049544340FFFFFF5F100E57C000000000B851434000000060091157C0000000E05353434000000020B21257C0FFFFFFDF48554340FFFFFF3FC11657C00000004060564340000000005F1657C0FFFFFF5FD95B4340000000A0491957C0FFFFFF9F355E4340 3.21533099999999994 +26 0106000020E6100000010000000103000000010000001D000000000000008BBD56C0000000608A6F43400000006075C756C0000000A0EE6F43400000000093C756C0FFFFFF3FF3764340FFFFFFBFD7CC56C0FFFFFF7F7277434000000020BECC56C0FFFFFFFF1B7F43400000000038D156C0000000A0817F434000000080F4D056C0000000605F924340000000205CCC56C00000008053924340000000E0F3CB56C0FFFFFF9F039D4340000000A0DEAE56C0FFFFFF7FC49C4340FFFFFFDFF6AD56C0000000601299434000000000DFAD56C00000008075924340000000802FAC56C000000060FE8B4340FFFFFF1F49AD56C000000060728743400000004030AD56C000000060D6844340FFFFFFDFCEAA56C000000060B6774340FFFFFFDF6EAC56C000000060557743400000002088AC56C000000060887543400000000071AD56C000000080B07543400000004047AD56C0000000C04C744340000000C0FDAE56C0000000406D754340000000C0DAAE56C0000000802B774340000000C070B256C0000000A0FE754340FFFFFFBF86B356C000000040A3744340000000A00EB456C0000000A0907043400000008058B856C000000000FB7143400000004020BC56C0000000C09A714340000000C058BD56C0000000C09E724340000000008BBD56C0000000608A6F4340 2.83366400000000018 +38 0106000020E6100000010000000103000000010000000E000000000000605FFD55C000000020A5494340000000808A0956C000000080C149434000000040780956C0000000005A4D434000000040011156C000000000AF4D4340000000404F1256C000000040734F434000000040D01256C0000000605C52434000000000BD1156C000000020A9544340000000407C1256C0FFFFFF7FA75E4340000000C0BE1056C0000000A0065F434000000080C61056C0000000A04C6D43400000002025FD55C0FFFFFF1F7F6D434000000080B9FA55C0000000607E6D434000000020A9FA55C00000008093494340000000605FFD55C000000020A5494340 1.00387500000000007 +31 0106000020E6100000010000000103000000010000001B000000FFFFFF5F100E57C000000000B851434000000040600C57C0000000C049544340000000A0C40C57C0000000801D57434000000000640E57C0000000002259434000000000060E57C000000000B45B4340FFFFFF3FFF0B57C000000020765E4340FFFFFF7F310957C0000000006468434000000000750A57C000000080796F434000000000260A57C0000000A03A714340000000C0E50A57C000000040E1724340000000C0F20957C000000020DE764340FFFFFFBFC70857C000000020EC87434000000020290757C0FFFFFF3F92884340000000A02EE956C0FFFFFF1F09884340000000A0D3E956C0000000202C5A434000000000D4E956C0000000C00D5A4340FFFFFF7F6BEF56C0000000A0605A4340FFFFFFFFACF056C000000040975843400000008054F356C0FFFFFFDFFB56434000000060C1F656C0000000E0725643400000000014FD56C000000000744C4340000000E005FF56C000000020904B434000000040220257C0000000405C484340000000E0880657C0FFFFFFDFF147434000000000EA0A57C0000000407C4A4340000000E0B10C57C000000020054D4340FFFFFF5F100E57C000000000B8514340 3.01748599999999989 +33 0106000020E6100000010000000103000000010000001400000000000060CA8756C00000002077664340000000603D8756C0000000C04C6A434000000000808856C0FFFFFF1F306D4340000000809C8F56C0000000A00E75434000000000DA9156C0000000205D764340FFFFFF1FDD9156C000000060B47F4340000000A0B78956C0000000A0BD7F4340FFFFFF3FF56C56C000000020977F434000000020526956C0000000E08A7F4340000000A03A6956C0000000C0966F4340000000A0D86656C0000000E0976F4340FFFFFF1FC26656C0FFFFFFBFE25E434000000000AD6656C0000000408054434000000060BA6D56C00000006031544340000000C0BF8B56C0000000A052544340FFFFFF7FF18C56C000000080A5594340000000A0948C56C0000000E0AA5C434000000020758A56C0000000E0F4624340000000C0A68856C0000000C08A64434000000060CA8756C00000002077664340 7.97395700000000041 +34 0106000020E61000000100000001030000000100000015000000FFFFFF5FFEBD56C0000000C0ED454340000000C0E0C056C0000000401048434000000040E0C356C0000000C0AC4D434000000060B0C556C000000000094E4340000000201BC956C0000000E0D74C4340FFFFFF3F1ECD56C0000000204D4E4340000000806BCE56C0000000600150434000000020DDCF56C0000000A015544340000000E0F7D256C0000000801D584340000000E067D556C000000080E65943400000004001D856C000000060795943400000000033DB56C0000000804A5B4340000000E0C8DA56C000000080986C434000000000A0D156C0000000A0FF6B43400000000038D156C0000000A0817F434000000020BECC56C0FFFFFFFF1B7F4340FFFFFFBFD7CC56C0FFFFFF7F727743400000000093C756C0FFFFFF3FF37643400000006075C756C0000000A0EE6F4340000000008BBD56C0000000608A6F4340FFFFFF5FFEBD56C0000000C0ED454340 5.00546399999999991 +35 0106000020E61000000100000001030000000100000031000000FFFFFF9F36AF56C0000000E037514340000000602AB256C0FFFFFFDFE74943400000008057B356C000000020CF4A434000000060A8B456C0000000C07D4A4340000000005FBA56C00000006033454340FFFFFF5FFEBD56C0000000C0ED454340000000008BBD56C0000000608A6F4340000000C058BD56C0000000C09E7243400000004020BC56C0000000C09A7143400000008058B856C000000000FB714340000000A00EB456C0000000A090704340FFFFFFBF86B356C000000040A3744340000000C070B256C0000000A0FE754340000000C0DAAE56C0000000802B774340000000C0FDAE56C0000000406D7543400000004047AD56C0000000C04C7443400000000071AD56C000000080B07543400000002088AC56C00000006088754340FFFFFFDF6EAC56C00000006055774340FFFFFFDFCEAA56C000000060B67743400000004024A856C0000000E0BD7043400000004080A456C0000000A0876F434000000080F2A156C00000004020724340000000E0119E56C000000060C67A4340000000C06F9A56C0000000A02D7B4340000000A0769456C0000000606376434000000000DA9156C0000000205D764340000000809C8F56C0000000A00E75434000000000808856C0FFFFFF1F306D4340000000603D8756C0000000C04C6A434000000060CA8756C0000000207766434000000060A28856C0000000A04C69434000000020CB8C56C0000000409A69434000000080AB9056C0000000608F6D4340FFFFFFFF6D9256C0000000204F714340000000C05C9456C000000040DD714340000000E0A39556C0000000A065704340000000400B9756C000000040A26A434000000040CC9956C0000000A0AA694340FFFFFFFFBA9B56C000000060646A4340000000A0EF9C56C0FFFFFF1FCB694340000000605F9F56C0000000202B614340000000402CA256C000000000995C4340000000C00CA356C00000006094584340000000A08CA656C0000000E04C57434000000000F0A856C000000020B5584340000000A086AB56C0FFFFFF5FCD5643400000004012AC56C0000000004C544340FFFFFF9F36AF56C0000000E037514340 2.46389099999999983 +37 0106000020E6100000010000000103000000010000002A0000000000006054AF56C0000000C0573B4340FFFFFF9F36AF56C0000000E0375143400000004012AC56C0000000004C544340000000A086AB56C0FFFFFF5FCD56434000000000F0A856C000000020B5584340000000A08CA656C0000000E04C574340000000C00CA356C00000006094584340000000402CA256C000000000995C4340000000605F9F56C0000000202B614340000000A0EF9C56C0FFFFFF1FCB694340FFFFFFFFBA9B56C000000060646A434000000040CC9956C0000000A0AA694340000000400B9756C000000040A26A4340000000E0A39556C0000000A065704340000000C05C9456C000000040DD714340FFFFFFFF6D9256C0000000204F71434000000080AB9056C0000000608F6D434000000020CB8C56C0000000409A69434000000060A28856C0000000A04C69434000000060CA8756C00000002077664340000000E0F48A56C000000020B1644340000000004D8C56C0000000C05E61434000000020578F56C0000000A0725D434000000020609356C000000020CC55434000000000419456C0000000A03D4A434000000000B89056C0000000C031444340000000A0029156C00000006064424340FFFFFF5F519356C0000000A0B336434000000020BC9556C00000004007324340000000406F9656C00000008005364340FFFFFF1FAA9556C0000000A07439434000000040129A56C0000000C07E3A4340FFFFFF1FE19A56C000000040623D4340FFFFFF3F2B9A56C000000000263E4340000000E0379A56C00000002001404340FFFFFFFFF5A556C0000000801A404340000000C049A756C0000000A07B3C4340000000801EAA56C000000020A63D4340FFFFFFDFD4AA56C000000060A538434000000060CFAB56C000000040BE384340000000202CAC56C000000040A33B43400000006054AF56C0000000C0573B4340 7.37797400000000003 +39 0106000020E61000000100000001030000000100000008000000000000A04E4956C0000000E03140434000000040454956C000000040E25E4340000000202B4956C000000080AC69434000000040F72C56C000000000486A434000000000EE2C56C000000060424E434000000080062D56C0FFFFFF7FAE3C4340000000803D4956C0FFFFFF9FB63C4340000000A04E4956C0000000E031404340 3.19004699999999985 +60 0106000020E6100000010000000103000000010000000A0000000000006098A956C0000000C019DE42400000006010C756C000000020A7DE4240000000A08DC656C000000000241A43400000002017B256C0000000209F1A434000000080FEAB56C0000000C0710E434000000080D3AB56C0FFFFFF5F2F0C434000000080FFA956C000000080EA0C43400000006018AA56C0000000E0FF0A434000000060E4A856C000000080440A43400000006098A956C0000000C019DE4240 3.26394699999999993 +41 0106000020E6100000010000000103000000010000001700000000000040454956C000000040E25E4340000000A04E4956C0000000E031404340000000A0FB5056C0000000001A414340000000E00D5356C0000000004C404340000000A0DA5656C000000040A4414340000000A07D5956C0000000A0833E4340000000608F5B56C0000000C0273F4340000000E0405D56C0000000803A3E4340000000E0CB5E56C000000040AD3B4340000000E08A6156C0000000A0793D434000000060B56256C0000000A09A3C4340000000A0A06456C0000000A0963D434000000020996756C000000060AB3B4340000000E0156856C0000000A085394340FFFFFF3F536956C0000000C06A384340FFFFFF5F876A56C0000000809638434000000080DD6A56C000000020BB364340000000806B6D56C0000000E09E35434000000060BA6D56C0000000603154434000000000AD6656C00000004080544340FFFFFF1FC26656C0FFFFFFBFE25E434000000060D25056C000000060FA5E434000000040454956C000000040E25E4340 2.44759700000000002 +42 0106000020E610000001000000010300000001000000210000000000000014FD56C000000000744C4340000000C0D70157C000000020CE464340000000E0CC0057C0000000A0ED404340FFFFFF1F3C0257C0000000A0CE3C434000000060C80457C0000000E02D3C4340000000A0020757C000000000763A4340000000602E0957C000000060EF3B434000000020B90A57C000000060DE3B434000000060340A57C0000000402D38434000000060310857C00000004012354340000000E0F50757C0000000E095324340000000E0D40857C000000080DD304340000000E0690B57C0FFFFFF3F3B30434000000000520C57C000000060732E4340000000208D0C57C000000000AB2A434000000060D80E57C000000080CC2A4340000000C05C1057C0000000209229434000000040131257C000000040C52A4340000000C0281A57C000000080262B4340000000E0D81957C000000000F6354340FFFFFF9FB91F57C0FFFFFFDF7D364340000000A0491957C0FFFFFF9F355E4340000000005F1657C0FFFFFF5FD95B4340FFFFFF3FC11657C0000000406056434000000020B21257C0FFFFFFDF4855434000000060091157C0000000E053534340FFFFFF5F100E57C000000000B8514340000000E0B10C57C000000020054D434000000000EA0A57C0000000407C4A4340000000E0880657C0FFFFFFDFF147434000000040220257C0000000405C484340000000E005FF56C000000020904B43400000000014FD56C000000000744C4340 1.29495800000000005 +44 0106000020E6100000010000000103000000010000001700000000000060850C57C000000080EC244340000000208D0C57C000000000AB2A434000000000520C57C000000060732E4340000000E0690B57C0FFFFFF3F3B304340000000E0D40857C000000080DD304340000000E0F50757C0000000E09532434000000060310857C0000000401235434000000060340A57C0000000402D38434000000020B90A57C000000060DE3B4340000000602E0957C000000060EF3B4340000000A0020757C000000000763A434000000060C80457C0000000E02D3C4340FFFFFF1F3C0257C0000000A0CE3C4340000000E0CC0057C0000000A0ED404340000000C0D70157C000000020CE4643400000000014FD56C000000000744C434000000060C1F656C0000000E0725643400000008054F356C0FFFFFFDFFB564340FFFFFFFFACF056C00000004097584340FFFFFF7F6BEF56C0000000A0605A434000000000D4E956C0000000C00D5A4340FFFFFF9FB0E956C0FFFFFF1F1425434000000060850C57C000000080EC244340 4.13399699999999992 +45 0106000020E610000001000000010300000001000000190000000000002017B256C0000000209F1A4340000000A08DC656C000000000241A43400000000009D656C0FFFFFFDF1C1A4340FFFFFF5F2CD856C0000000E0381A43400000004025D856C000000020FA1A4340000000E0A7D756C0000000E0553543400000004001D856C00000006079594340000000E067D556C000000080E6594340000000E0F7D256C0000000801D58434000000020DDCF56C0000000A015544340000000806BCE56C00000006001504340FFFFFF3F1ECD56C0000000204D4E4340000000201BC956C0000000E0D74C434000000060B0C556C000000000094E434000000040E0C356C0000000C0AC4D4340000000C0E0C056C00000004010484340FFFFFF5FFEBD56C0000000C0ED454340000000005FBA56C0000000603345434000000060A8B456C0000000C07D4A43400000008057B356C000000020CF4A4340000000602AB256C0FFFFFFDFE7494340FFFFFF9F36AF56C0000000E0375143400000006054AF56C0000000C0573B43400000006067AF56C000000040593243400000002017B256C0000000209F1A4340 4.29831099999999999 +46 0106000020E6100000010000000103000000010000001200000000000060DE7956C0000000E08D1C4340000000C0897B56C0000000806323434000000060177B56C0000000807E244340000000204D7A56C0000000A0CD234340FFFFFF7FB97A56C0000000409227434000000000028256C0000000E0EB27434000000020048256C0000000002F2A4340000000A04E8956C00000002053344340000000C05A8956C000000020A5364340000000A0029156C0000000606442434000000000B89056C0000000C031444340000000A06B8F56C0000000000A484340000000E0C18B56C0000000601D4E4340000000C0BF8B56C0000000A05254434000000060BA6D56C00000006031544340000000806B6D56C0000000E09E35434000000080B86D56C0FFFFFFFF081C434000000060DE7956C0000000E08D1C4340 27.4838270000000016 +48 0106000020E6100000010000000103000000010000000E000000FFFFFF9FD60956C00000000035214340000000808A0956C000000080C1494340000000605FFD55C000000020A549434000000060C8FC55C0FFFFFF5FDF444340000000C0AAFD55C0000000808E3F434000000080F9FC55C000000060DA394340000000E068FD55C000000060D537434000000000E6FC55C0000000C05E364340000000E0A8FE55C0000000C056334340FFFFFFFFA6FE55C00000004061304340000000E0ACFD55C0000000E0D02C4340000000407CFD55C000000040D92543400000006042FF55C00000004040214340FFFFFF9FD60956C00000000035214340 0 +49 0106000020E6100000010000000103000000010000001600000000000060DE7956C0000000E08D1C434000000020918256C0000000809A1C4340000000207D8256C0000000008611434000000020488D56C000000040610B434000000080429056C000000040A30F434000000060899256C0000000405A154340000000C08C9556C0000000C02718434000000060589756C000000080FD1D434000000060A39756C0000000606A294340FFFFFFBFF49656C0FFFFFF1FC32E434000000020BC9556C00000004007324340FFFFFF5F519356C0000000A0B3364340000000A0029156C00000006064424340000000C05A8956C000000020A5364340000000A04E8956C0000000205334434000000020048256C0000000002F2A434000000000028256C0000000E0EB274340FFFFFF7FB97A56C00000004092274340000000204D7A56C0000000A0CD23434000000060177B56C0000000807E244340000000C0897B56C0000000806323434000000060DE7956C0000000E08D1C4340 2.93446600000000002 +50 0106000020E6100000010000000103000000010000001600000000000020396656C0000000A0C31B434000000080B86D56C0FFFFFFFF081C4340000000806B6D56C0000000E09E35434000000080DD6A56C000000020BB364340FFFFFF5F876A56C00000008096384340FFFFFF3F536956C0000000C06A384340000000E0156856C0000000A08539434000000020996756C000000060AB3B4340000000A0A06456C0000000A0963D434000000060B56256C0000000A09A3C4340000000E08A6156C0000000A0793D4340000000E0CB5E56C000000040AD3B4340000000E0405D56C0000000803A3E4340000000608F5B56C0000000C0273F4340000000A07D5956C0000000A0833E4340000000A0DA5656C000000040A4414340000000E00D5356C0000000004C404340000000A0FB5056C0000000001A414340000000A04E4956C0000000E031404340000000803D4956C0FFFFFF9FB63C434000000080A84956C0000000A05B1B434000000020396656C0000000A0C31B4340 4.45642699999999969 +51 0106000020E6100000010000000103000000010000002400000000000060E4A856C000000080440A43400000006018AA56C0000000E0FF0A434000000080FFA956C000000080EA0C434000000080D3AB56C0FFFFFF5F2F0C434000000080FEAB56C0000000C0710E43400000002017B256C0000000209F1A43400000006067AF56C000000040593243400000006054AF56C0000000C0573B4340000000202CAC56C000000040A33B434000000060CFAB56C000000040BE384340FFFFFFDFD4AA56C000000060A5384340000000801EAA56C000000020A63D4340000000C049A756C0000000A07B3C4340FFFFFFFFF5A556C0000000801A404340000000E0379A56C00000002001404340FFFFFF3F2B9A56C000000000263E4340FFFFFF1FE19A56C000000040623D434000000040129A56C0000000C07E3A4340FFFFFF1FAA9556C0000000A074394340000000406F9656C0000000800536434000000020BC9556C00000004007324340FFFFFFBFF49656C0FFFFFF1FC32E434000000060A39756C0000000606A29434000000060589756C000000080FD1D4340000000C08C9556C0000000C02718434000000060899256C0000000405A15434000000080429056C000000040A30F434000000060029356C000000020C60B4340FFFFFFDF129556C000000060CA0C4340000000E0919A56C0000000E0CE054340FFFFFF7F92A656C0FFFFFFBF5400434000000060F6A756C0000000E03B014340FFFFFF9F39A756C0000000E0A3024340000000E08CA756C0000000A00E064340000000A003A756C0000000C05F09434000000060E4A856C000000080440A4340 4.62926400000000005 +53 0106000020E6100000010000000103000000010000001F00000000000040E21957C0000000A0A6024340000000E0092157C0000000E02C03434000000000242157C0000000608104434000000060892357C0000000405C064340FFFFFF7FC12357C0000000A0EA074340000000A0D82457C00000000027084340FFFFFF7FDB2457C0000000E0450C434000000080C22557C0000000A0750C434000000020B82557C000000040160E434000000020652657C0000000402B0E434000000020632657C0000000A05C114340FFFFFF9F102757C00000006080114340000000A0F62657C0FFFFFFDF91154340000000C0FE2857C0FFFFFF3FEE154340000000A0342957C000000080811A4340FFFFFFDFD72C57C0000000A03B1C4340000000C0662C57C000000040292C434000000060CC2857C0FFFFFFBF582C4340000000800C2857C0FFFFFFBF01374340FFFFFF9FB91F57C0FFFFFFDF7D364340000000E0D81957C000000000F6354340000000C0281A57C000000080262B434000000040131257C000000040C52A4340000000C05C1057C0000000209229434000000060D80E57C000000080CC2A4340000000208D0C57C000000000AB2A434000000060850C57C000000080EC244340000000A0C10C57C00000008020154340000000E0AA0B57C00000004000154340FFFFFF1F090C57C0000000202E02434000000040E21957C0000000A0A6024340 3.99004100000000017 +55 0106000020E6100000010000000103000000010000002200000000000000181856C000000020FEF4424000000020F21756C000000040DA204340FFFFFF9FD60956C000000000352143400000006042FF55C00000004040214340FFFFFF9FB8FE55C0000000E0DB1E4340000000C01AFF55C0000000600E1E43400000006096FE55C0FFFFFFFFB0194340000000A0AAFB55C0000000A0E7154340FFFFFFBFA5FB55C0000000E02914434000000020D6FC55C00000006086114340000000E04DFE55C000000080DD104340000000E02F0156C000000000390D434000000000CA0056C000000000D20B434000000060C0FD55C000000040620C4340000000406BFE55C0000000206209434000000000390256C000000040EC06434000000000C20256C000000080C605434000000080A70256C000000020E704434000000080630156C0000000C04A044340000000A0DE0156C0000000E00D014340000000A0630156C0FFFFFF9FCEFC424000000080B80256C0000000E066FA4240FFFFFF5FAC0256C0000000A09DF74240FFFFFFBF220456C00000002003F74240000000600D0556C000000000D5F8424000000040600556C0000000803AF64240000000C0F20156C0000000A073F54240000000A0B30156C0000000E0EFF34240FFFFFF1FDF0256C000000040B0F24240FFFFFFBF670656C0FFFFFF5FFDF34240000000407E0656C00000006099F24240000000A0390956C0000000E0E8F54240000000E0CA0956C0000000201CF5424000000000181856C000000020FEF44240 3.04025299999999987 +56 0106000020E6100000010000000103000000010000000600000000000000181856C000000020FEF4424000000020572D56C0FFFFFF7F74F44240000000E0562D56C0000000208910434000000060492D56C0000000603E21434000000020F21756C000000040DA20434000000000181856C000000020FEF44240 3.90541099999999997 +57 0106000020E6100000010000000103000000010000001700000000000020488D56C000000040610B4340000000207D8256C0000000008611434000000020918256C0000000809A1C434000000060DE7956C0000000E08D1C434000000080B86D56C0FFFFFFFF081C434000000020396656C0000000A0C31B434000000000936656C0000000601DFA424000000040B46A56C0000000607CEB424000000060E56B56C0FFFFFFFF16EA4240000000403A6C56C00000008003E74240000000E09E6E56C0000000A0A5EB424000000080827656C000000020D9F34240FFFFFF5F1B7756C000000000E7F34240FFFFFFDF797756C00000000029F24240FFFFFF9FA27956C0000000A01DF0424000000020067C56C0000000C063F0424000000080A67E56C0000000A0B8F44240000000A0537D56C00000006058FB424000000020B18056C0000000A012FC4240000000E0AE8256C0FFFFFF5F21FF424000000040A38756C00000008021044340000000209F8856C0000000E0E706434000000020488D56C000000040610B4340 4.33283899999999988 +58 0106000020E6100000010000000103000000010000000B00000000000020864856C00000000054104340000000C0FA4856C0FFFFFF3FC70D434000000000E64756C000000060EA0B4340FFFFFFFFF54856C0000000E00F06434000000080504956C0000000C0DEFE4240000000E0744B56C0000000A07CF9424000000000936656C0000000601DFA424000000020396656C0000000A0C31B434000000080A84956C0000000A05B1B4340FFFFFF9FB84956C0000000409110434000000020864856C00000000054104340 3.8941110000000001 +61 0106000020E61000000100000001030000000100000013000000000000A0F60157C0000000C081014340000000405FFE56C0000000E06B014340000000605AFD56C0000000E05905434000000060BBFB56C0000000809A044340000000C01EFB56C00000002014064340FFFFFFDFD2F956C000000040E2064340FFFFFF3FF2E856C0000000A0A3064340000000A0D7E856C0FFFFFF1F1B14434000000060BCE256C0000000C023144340FFFFFFFF04E256C0FFFFFFBFD01343400000008044E256C0000000E0BEE44240000000803CF456C000000040BEE4424000000080DBF356C0FFFFFF9F97DF42400000004063F456C0FFFFFFBF64DB4240000000007EF456C000000000A6CC4240000000A0000257C00000002054CD424000000080730157C0000000C094E3424000000080EA0157C0000000409BE44240000000A0F60157C0000000C081014340 3.28216300000000016 +62 0106000020E6100000010000000103000000010000000C00000000000020572D56C0FFFFFF7F74F4424000000020682D56C0FFFFFF7F00EF4240000000C0E94956C000000080CDEE4240000000E0EC4956C0000000607AF94240000000E0744B56C0000000A07CF9424000000080504956C0000000C0DEFE4240FFFFFFFFF54856C0000000E00F06434000000000E64756C000000060EA0B4340000000C0FA4856C0FFFFFF3FC70D434000000020864856C00000000054104340000000E0562D56C0000000208910434000000020572D56C0FFFFFF7F74F44240 3.29576199999999986 +63 0106000020E61000000100000001030000000100000013000000000000E0718756C00000000011D64240FFFFFF5F418A56C0FFFFFF3F72DA4240FFFFFFDFFD8C56C00000004087D64240000000E0A59D56C0FFFFFF5FA4F0424000000000CE9456C0000000603BFE4240000000E0919A56C0000000E0CE054340FFFFFFDF129556C000000060CA0C434000000060029356C000000020C60B434000000080429056C000000040A30F434000000020488D56C000000040610B4340000000209F8856C0000000E0E706434000000040A38756C00000008021044340000000E0AE8256C0FFFFFF5F21FF424000000020B18056C0000000A012FC4240000000A0537D56C00000006058FB424000000080A67E56C0000000A0B8F4424000000020067C56C0000000C063F04240000000E0798056C0000000E0DEE84240000000E0718756C00000000011D64240 7.24967900000000043 +64 0106000020E6100000010000000103000000010000001100000000000060BC8956C00000004054D24240FFFFFF7F82A256C00000008046D2424000000040D6A956C0000000C024D242400000006098A956C0000000C019DE424000000060E4A856C000000080440A4340000000A003A756C0000000C05F094340000000E08CA756C0000000A00E064340FFFFFF9F39A756C0000000E0A302434000000060F6A756C0000000E03B014340FFFFFF7F92A656C0FFFFFFBF54004340000000E0919A56C0000000E0CE05434000000000CE9456C0000000603BFE4240000000E0A59D56C0FFFFFF5FA4F04240FFFFFFDFFD8C56C00000004087D64240FFFFFF5F418A56C0FFFFFF3F72DA4240000000E0718756C00000000011D6424000000060BC8956C00000004054D24240 3.04184600000000005 +65 0106000020E6100000010000000103000000010000000B000000000000E0F40F57C0000000C0B5CD4240FFFFFF9FF10F57C0000000000DD34240000000403A1A57C0000000E038DB4240000000001E1A57C00000002044EE424000000040E21957C0000000A0A6024340FFFFFF1F090C57C0000000202E024340000000A0F60157C0000000C08101434000000080EA0157C0000000409BE4424000000080730157C0000000C094E34240000000A0000257C00000002054CD4240000000E0F40F57C0000000C0B5CD4240 1.61801799999999996 +66 0106000020E6100000010000000103000000010000001400000000000000CE4956C00000004053CD4240000000C0655D56C0000000A09ECD4240FFFFFF7F825D56C0FFFFFF1FA9CA4240FFFFFF7F835E56C000000080FAC8424000000020996156C0000000E035C9424000000000DB6056C0FFFFFFBFD6CE4240000000403A6156C0000000803FD3424000000020DB6056C00000000005D74240000000A0606156C000000020EFD84240FFFFFF3F366556C0000000A061DA424000000040A76A56C0000000006BDF424000000040416B56C00000002059E44240000000403A6C56C00000008003E7424000000060E56B56C0FFFFFFFF16EA424000000040B46A56C0000000607CEB424000000000936656C0000000601DFA4240000000E0744B56C0000000A07CF94240000000E0EC4956C0000000607AF94240000000C0E94956C000000080CDEE424000000000CE4956C00000004053CD4240 4.91080100000000019 +67 0106000020E6100000010000000103000000010000002000000000000060BC8956C00000004054D24240000000E0718756C00000000011D64240000000E0798056C0000000E0DEE8424000000020067C56C0000000C063F04240FFFFFF9FA27956C0000000A01DF04240FFFFFFDF797756C00000000029F24240FFFFFF5F1B7756C000000000E7F3424000000080827656C000000020D9F34240000000E09E6E56C0000000A0A5EB4240000000403A6C56C00000008003E7424000000040416B56C00000002059E4424000000040A76A56C0000000006BDF4240FFFFFF3F366556C0000000A061DA4240000000A0606156C000000020EFD8424000000020DB6056C00000000005D74240000000403A6156C0000000803FD3424000000000DB6056C0FFFFFFBFD6CE424000000020996156C0000000E035C94240000000A0D56556C0FFFFFF3F7FC9424000000020FD6656C00000006065CC4240000000A08C6856C000000080A3CB424000000020126956C000000000DBCC424000000060DB6B56C0000000801BCB4240000000E03A6C56C00000008035CC424000000020756D56C000000020BACC4240000000A0296E56C000000080E9CB4240000000C0F16E56C0000000E0A9CC4240000000E0817156C0FFFFFF3F6FCB4240000000A0D07356C000000080EECC424000000060627756C000000060F8CC424000000000908956C0000000E0B3CC424000000060BC8956C00000004054D24240 1.99145700000000003 +69 0106000020E6100000010000000103000000010000001100000000000060E6E956C000000040FFB54240000000C0B1F056C00000006063B6424000000060EAF056C0000000E033CC4240000000007EF456C000000000A6CC42400000004063F456C0FFFFFFBF64DB424000000080DBF356C0FFFFFF9F97DF4240000000803CF456C000000040BEE442400000008044E256C0000000E0BEE442400000000065D456C00000004041E44240000000C086D456C000000040CED94240000000C080CA56C0000000001BD942400000008089CA56C00000008095CB4240000000408AD456C000000000C3CB4240000000A067D456C000000060BDC042400000008032CE56C00000006039C04240000000C049CE56C0000000E0D8B4424000000060E6E956C000000040FFB54240 7.26665000000000028 +70 0106000020E61000000100000001030000000100000014000000FFFFFF7F82A256C00000008046D24240FFFFFF1FC4A256C00000000069CC4240000000E078A356C00000002057CC4240000000408CA356C0000000C0F4A74240FFFFFF7FD8A356C0000000A004A34240000000808AAF56C0000000E0CBA24240FFFFFF3F5BB056C000000020F4A24240000000800DB056C00000002043AF424000000040D5B156C00000004066AF424000000080CAB156C0000000600DCD4240FFFFFF7F5EC056C0000000805BCD4240000000C00AC756C00000000081CB42400000008089CA56C00000008095CB4240000000C080CA56C0000000001BD9424000000060FBC956C0000000A01ED94240FFFFFF9FF7C956C0FFFFFF7F75DE42400000006010C756C000000020A7DE42400000006098A956C0000000C019DE424000000040D6A956C0000000C024D24240FFFFFF7F82A256C00000008046D24240 3.11090400000000011 +71 0106000020E6100000010000000103000000010000000900000000000060378E56C0000000A0EBA74240000000408CA356C0000000C0F4A74240000000E078A356C00000002057CC4240FFFFFF1FC4A256C00000000069CC4240FFFFFF7F82A256C00000008046D2424000000060BC8956C00000004054D2424000000000908956C0000000E0B3CC424000000040968956C000000040EAA7424000000060378E56C0000000A0EBA74240 2.98027100000000011 +72 0106000020E6100000010000000103000000010000000D00000000000080CD0557C0000000407487424000000000A11057C000000060E7874240000000A0F00F57C000000080AAA04240FFFFFF7F761057C0000000A0EEA0424000000000221057C0000000C028BD4240000000E0F40F57C0000000C0B5CD4240000000A0000257C00000002054CD4240000000007EF456C000000000A6CC424000000060EAF056C0000000E033CC4240000000C0B1F056C00000006063B6424000000060E6E956C000000040FFB5424000000000B0EA56C0000000004286424000000080CD0557C00000004074874240 3.86676699999999984 +73 0106000020E6100000010000000103000000010000000E000000000000E0FF5D56C0000000C071AB424000000020E35B56C00000002088AD4240000000605D5B56C0000000409CB4424000000020085D56C00000000002BA424000000080AA5F56C0000000E0F0BE424000000020996156C0000000E035C94240FFFFFF7F835E56C000000080FAC84240FFFFFF7F825D56C0FFFFFF1FA9CA4240000000C0655D56C0000000A09ECD424000000000CE4956C00000004053CD4240000000E0F64256C0FFFFFFDF38CD4240000000002E4356C0FFFFFFFF29AB4240000000C0BC4F56C0000000C03CAB4240000000E0FF5D56C0000000C071AB4240 1.86840800000000007 +74 0106000020E61000000100000001030000000100000024000000000000808AAF56C0000000E0CBA24240000000A093AF56C0000000E0549542400000006089B056C00000002032954240000000E0B1B056C0FFFFFFFF0E924240000000202CB256C00000002007924240000000C071B256C000000060B4864240FFFFFFBF2FBE56C0000000A04E874240000000802ABE56C000000080B48C4240FFFFFF7F90C156C0FFFFFF9FCB8C424000000080BAC156C0000000C003924240000000E09DC256C0000000A01B92424000000020C3C256C0000000807895424000000020D4C456C00000008020954240000000C0EAC456C0FFFFFF7F84964240000000E019C656C0000000409A964240000000E035C656C000000080E899424000000000C3C856C000000000D7994240000000A0D9C856C0000000C0899E42400000006055C856C0FFFFFF1FAB9E42400000000047C856C0000000804BA04240000000005BCA56C0000000A0D1A04240000000805DCA56C0000000204EA84240000000E096CB56C0000000E080A8424000000080B0CB56C000000040A1B44240000000C049CE56C0000000E0D8B442400000008032CE56C00000006039C04240000000A067D456C000000060BDC04240000000408AD456C000000000C3CB42400000008089CA56C00000008095CB4240000000C00AC756C00000000081CB4240FFFFFF7F5EC056C0000000805BCD424000000080CAB156C0000000600DCD424000000040D5B156C00000004066AF4240000000800DB056C00000002043AF4240FFFFFF3F5BB056C000000020F4A24240000000808AAF56C0000000E0CBA24240 12.5770339999999994 +75 0106000020E6100000010000000103000000010000000D000000000000A0A47756C0000000C0DA904240000000E07A7D56C000000060D090424000000060B67D56C00000000057884240000000E0BD7F56C00000008017884240000000C0D87F56C0000000805986424000000040568756C00000000039864240000000605B8756C0000000E0188B4240000000E02F8E56C0000000A0058B424000000060378E56C0000000A0EBA7424000000040968956C000000040EAA7424000000000908956C0000000E0B3CC424000000060627756C000000060F8CC4240000000A0A47756C0000000C0DA904240 7.80359900000000017 +76 0106000020E6100000010000000103000000010000002800000000000060627756C000000060F8CC4240000000A0D07356C000000080EECC4240000000E0817156C0FFFFFF3F6FCB4240000000C0F16E56C0000000E0A9CC4240000000A0296E56C000000080E9CB424000000020756D56C000000020BACC4240000000E03A6C56C00000008035CC424000000060DB6B56C0000000801BCB424000000020126956C000000000DBCC4240000000A08C6856C000000080A3CB424000000020FD6656C00000006065CC4240000000A0D56556C0FFFFFF3F7FC9424000000020996156C0000000E035C9424000000080AA5F56C0000000E0F0BE424000000020085D56C00000000002BA4240000000605D5B56C0000000409CB4424000000020E35B56C00000002088AD4240000000E0FF5D56C0000000C071AB424000000080096056C0FFFFFF1F2BAA424000000080E36056C00000000009A7424000000080E36056C0FFFFFF1F61A3424000000080555F56C0000000A0C4A0424000000060D06056C000000000079F4240000000C0F46556C000000000279E424000000000FE6556C0000000A01B9D424000000040236856C000000020EE9C4240FFFFFF7F196856C000000000C59B424000000000CD6956C0000000E06A9B424000000060CC6956C00000008027994240000000A0336C56C00000000008994240FFFFFFFF326C56C0FFFFFF9F2C974240FFFFFF7F876D56C0FFFFFF5FD296424000000020906D56C0000000208094424000000020866E56C0000000806194424000000080986E56C0000000E0FC924240000000A0A07056C000000080BF924240000000A0B27056C000000060B790424000000060BB7156C000000060A7904240000000A0A47756C0000000C0DA90424000000060627756C000000060F8CC4240 3.47149000000000019 +77 0106000020E6100000010000000103000000010000001800000000000000B0EA56C0000000004286424000000060E6E956C000000040FFB54240000000C049CE56C0000000E0D8B4424000000080B0CB56C000000040A1B44240000000E096CB56C0000000E080A84240000000805DCA56C0000000204EA84240000000005BCA56C0000000A0D1A042400000000047C856C0000000804BA042400000006055C856C0FFFFFF1FAB9E4240000000A0D9C856C0000000C0899E424000000000C3C856C000000000D7994240000000E035C656C000000080E8994240000000E019C656C0000000409A964240000000C0EAC456C0FFFFFF7F8496424000000020D4C456C0000000802095424000000020C3C256C00000008078954240000000E09DC256C0000000A01B92424000000080BAC156C0000000C003924240FFFFFF7F90C156C0FFFFFF9FCB8C4240000000E031C756C000000080E88A4240000000806ECE56C0000000E0E18A424000000040AFCE56C000000020DF704240000000C0B7EA56C0000000C06A71424000000000B0EA56C00000000042864240 4.33482199999999995 +78 0106000020E6100000010000000103000000010000001F000000000000E08FAB56C0000000807A76424000000040ABAC56C000000080A176424000000020BFAC56C0000000A0BC7B4240FFFFFFDFF6AD56C000000000E37B424000000000FEAD56C0000000604F7F42400000004052AF56C000000000847F42400000000061AF56C0FFFFFFBF6B864240000000C071B256C000000060B4864240000000202CB256C00000002007924240000000E0B1B056C0FFFFFFFF0E9242400000006089B056C00000002032954240000000A093AF56C0000000E054954240000000808AAF56C0000000E0CBA24240FFFFFF7FD8A356C0000000A004A34240000000408CA356C0000000C0F4A7424000000060378E56C0000000A0EBA74240000000E02F8E56C0000000A0058B4240000000605B8756C0000000E0188B424000000040568756C0000000003986424000000060BC8956C0000000604186424000000060E68956C0000000809C81424000000060D28A56C0000000207C81424000000000F58A56C000000060D07E424000000000108C56C0000000A0A07E424000000060338C56C000000040987C4240FFFFFFFF318D56C000000080777C424000000040438D56C0000000A0217B4240000000607A8E56C000000080E27A424000000000938E56C000000080EC77424000000040D09056C00000004054764240000000E08FAB56C0000000807A764240 8.45153700000000008 +\. + + +CREATE INDEX getis_data_gix ON getis_data USING GIST(the_geom); + From 1d09eac3e7f8ef2ead4c2a7c2bf89db502aab4c5 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 16:10:29 -0400 Subject: [PATCH 21/48] adding pgsql tests --- src/pg/test/expected/16_getis_test.out | 23 +++++++++++++++++++++++ src/pg/test/sql/16_getis_test.sql | 13 +++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 src/pg/test/expected/16_getis_test.out create mode 100644 src/pg/test/sql/16_getis_test.sql diff --git a/src/pg/test/expected/16_getis_test.out b/src/pg/test/expected/16_getis_test.out new file mode 100644 index 0000000..ba5ee87 --- /dev/null +++ b/src/pg/test/expected/16_getis_test.out @@ -0,0 +1,23 @@ +\pset format unaligned +\set ECHO all +\i test/fixtures/ppoints.sql +SET client_min_messages TO WARNING; +\set ECHO none +_cdb_random_seeds + +(1 row) +z_score|p_value +-1.0358|0.0160 +-0.6404|0.0070 +2.6802|0.0190 +4.7529|0.0020 +2.7270|0.0250 +-1.2444|0.0230 +4.6322|0.0150 +4.5325|0.0220 +-1.3115|0.0090 +-1.3115|0.0080 +4.6361|0.0010 +2.8307|0.0310 +-1.3115|0.0020 +(13 rows) diff --git a/src/pg/test/sql/16_getis_test.sql b/src/pg/test/sql/16_getis_test.sql new file mode 100644 index 0000000..38977a3 --- /dev/null +++ b/src/pg/test/sql/16_getis_test.sql @@ -0,0 +1,13 @@ +\pset format unaligned +\set ECHO all +\i test/fixtures/getis_data.sql + +-- set random seed +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- test against PySAL example dataset +SELECT z_score, p_val +FROM cdb_crankshaft.CDB_GetisOrdsG( + 'select * from ppoints2', + 'ratio') As cdb_getisordsg(z_score, p_val, p_z_sim) +WHERE p_val <= 0.05; From dcb364c3eec992bed6a2f3681d693f4306ccc457 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 17:16:53 -0400 Subject: [PATCH 22/48] up default number of permutations --- src/pg/sql/16_getis.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index dc3a25d..6e52d50 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION column_name TEXT, w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, + permutations INT DEFAULT 999, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, p_z_sim NUMERIC, rowid BIGINT) From eff548dec92ceaef5ff90b5afa901434352686f6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 19 Sep 2016 17:17:51 -0400 Subject: [PATCH 23/48] aligning parameters for fistures and tests --- src/pg/test/sql/16_getis_test.sql | 9 ++- src/py/crankshaft/test/fixtures/getis.json | 79 +------------------ .../crankshaft/test/test_clustering_getis.py | 19 +++++ 3 files changed, 25 insertions(+), 82 deletions(-) diff --git a/src/pg/test/sql/16_getis_test.sql b/src/pg/test/sql/16_getis_test.sql index 38977a3..03cfcc6 100644 --- a/src/pg/test/sql/16_getis_test.sql +++ b/src/pg/test/sql/16_getis_test.sql @@ -6,8 +6,9 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); -- test against PySAL example dataset -SELECT z_score, p_val +SELECT z_score, p_value FROM cdb_crankshaft.CDB_GetisOrdsG( - 'select * from ppoints2', - 'ratio') As cdb_getisordsg(z_score, p_val, p_z_sim) -WHERE p_val <= 0.05; + 'select * from getis_data', + 'hr8893', 'knn', 5, 999, + 'the_geom', 'cartodb_id') As cdb_getisordsg(z_score, p_value, p_z_sim) +WHERE p_value <= 0.05; diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json index 71d7728..4d5837f 100644 --- a/src/py/crankshaft/test/fixtures/getis.json +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -1,78 +1 @@ -[[-0.37378594228210338, 0.436, 0.35428178954558454], - [-0.95225687055925445, 0.065000000000000002, 0.17048336261993491], - [-0.69155363829052829, 0.313, 0.2446088425605718], - [-0.91005743729633215, 0.085999999999999993, 0.18139610975939735], - [-1.0132869736739361, 0.13200000000000001, 0.1554615538579156], - [-0.51711928006609076, 0.058999999999999997, 0.30253644777604194], - [-0.59915606178126646, 0.125, 0.27453440990643974], - [-0.65270328962244883, 0.314, 0.25697379065855031], - [-1.035818892716347, 0.016, 0.15014332094164029], - [-0.77022454873249124, 0.14899999999999999, 0.22058335212423197], - [-0.33280112478125556, 0.40500000000000003, 0.36964220456990138], - [-0.16619696958659569, 0.28000000000000003, 0.43400097210156352], - [-1.0189197578270577, 0.114, 0.15412053143437432], - [-0.58736356248618182, 0.378, 0.27847978008948393], - [-0.60815535737753856, 0.34699999999999998, 0.27154222002447759], - [-0.83654622542342216, 0.22, 0.20142384153812443], - [-0.27623528229314609, 0.48099999999999998, 0.39118367986278957], - [-0.17751012385821857, 0.40600000000000003, 0.42955385691066428], - [-0.42339868568527872, 0.499, 0.33600220182364238], - [-0.81090507212288232, 0.17000000000000001, 0.2087100936430748], - [-0.63864423309742102, 0.089999999999999997, 0.26152719947142589], - [-0.64036180904066409, 0.0070000000000000001, 0.2609687028452613], - [-0.62209633811403076, 0.217, 0.26693926135549806], - [2.6802178678493869, 0.019, 0.0036787128295844296], - [-0.53304848636954871, 0.314, 0.29700000849115427], - [-0.15325607655300719, 0.33900000000000002, 0.43909816861874729], - [-0.46599673142949988, 0.5, 0.32060892008190012], - [-0.67328453973424696, 0.34000000000000002, 0.25038314326507338], - [-0.65374072057172605, 0.27300000000000002, 0.25663943186717253], - [-0.042646376611931783, 0.33900000000000002, 0.48299171295314569], - [-0.50268629316736713, 0.36899999999999999, 0.30759242403716802], - [0.20061122903056294, 0.41199999999999998, 0.42050128853782709], - [4.7529210422438366, 0.002, 1.0024933114749501e-06], - [-0.17112791409785774, 0.45500000000000002, 0.43206159366732011], - [2.7269760223697093, 0.025000000000000001, 0.0031958841790815651], - [-1.2443986350856617, 0.023, 0.1066764425086173], - [2.6146417757460672, 0.058999999999999997, 0.0044660540776073621], - [-0.81089418923393874, 0.217, 0.20871321876749027], - [-0.96842037905996448, 0.051999999999999998, 0.16641723288102916], - [4.632217640632744, 0.014999999999999999, 1.8088477418132243e-06], - [1.7098997741760702, 0.066000000000000003, 0.043642204110986293], - [-0.68576671544237189, 0.30399999999999999, 0.24643011428027894], - [-0.10705643473845843, 0.46100000000000002, 0.45737210425657626], - [-0.54280332475246651, 0.29499999999999998, 0.29363261191793877], - [0.031036400804321389, 0.26000000000000001, 0.48762025500133288], - [4.5324686370772209, 0.021999999999999999, 2.9149167000142029e-06], - [-1.3114835366024926, 0.0089999999999999993, 0.09484722615634944], - [-1.3114835366024926, 0.0080000000000000002, 0.09484722615634944], - [4.6361327065826368, 0.001, 1.7749405375466765e-06], - [1.4367662910941781, 0.081000000000000003, 0.075392205783569644], - [2.8306701087322108, 0.031, 0.0023225301610166893], - [-0.26132309263094761, 0.47599999999999998, 0.39692167988511706], - [-0.78055710491931951, 0.13400000000000001, 0.21753151443265129], - [-0.64234747392787916, 0.33300000000000002, 0.26032379813179585], - [-1.3114835366024926, 0.002, 0.09484722615634944], - [-0.57318180672306673, 0.26400000000000001, 0.28326080073223758], - [1.4602423896649199, 0.085999999999999993, 0.072111734510341252], - [-0.10852292180607989, 0.33700000000000002, 0.45679044945885094], - [0.25007822707788863, 0.25800000000000001, 0.4012634266959908], - [-0.15535540716918589, 0.36799999999999999, 0.43827056859443592], - [-0.15387021291338146, 0.34300000000000003, 0.43885603550771368], - [-0.23212287899595097, 0.434, 0.40822128763053078], - [-0.34331698296028734, 0.255, 0.36568000291448455], - [-0.21917003612352226, 0.38300000000000001, 0.41325879787572739], - [-0.79956517297385543, 0.20000000000000001, 0.21198138620767248], - [-0.57435424239923683, 0.20100000000000001, 0.28286405516217616], - [0.17090357402483336, 0.17599999999999999, 0.43214979317279412], - [-0.45071423902325392, 0.39500000000000002, 0.32609775926146312], - [0.76813695849036678, 0.16500000000000001, 0.22120291089357114], - [0.43164294033005113, 0.14899999999999999, 0.33300047214912831], - [0.3756136313798501, 0.16, 0.35360207763890095], - [-0.35027449210479344, 0.47599999999999998, 0.36306635326063197], - [-0.48157903675663827, 0.45500000000000002, 0.31505251051764338], - [0.82850242294957743, 0.187, 0.20369301166122333], - [0.32654794875356141, 0.23599999999999999, 0.3720049088953894], - [-0.30807434754194035, 0.497, 0.37901288012280243], - [0.50368346846574807, 0.159, 0.30724191359295261], - [0.75454835573966283, 0.13500000000000001, 0.22526001043744448]] +[[-0.37337341596824414, 0.43099999999999999, 0.35443527112672846], [-0.95217153199599269, 0.091999999999999998, 0.17050499804382691], [-0.69132084390888426, 0.30299999999999999, 0.24468196785399454], [-0.90994823039851658, 0.088999999999999996, 0.18142490623820451], [-1.0132361542216282, 0.129, 0.15547368761207747], [-0.51678782422627401, 0.058999999999999997, 0.30265214024759912], [-0.59887100659663139, 0.13400000000000001, 0.27462945349842682], [-0.65244852117509244, 0.32900000000000001, 0.2570559358979031], [-1.0357808174963727, 0.0070000000000000001, 0.1501522043425858], [-0.77003625124250874, 0.13600000000000001, 0.22063919456432224], [-0.33236541712916268, 0.375, 0.3698066743983055], [-0.16566702930134228, 0.28699999999999998, 0.43420949712949208], [-1.0188721243221961, 0.113, 0.15413183958086896], [-0.58707183737030266, 0.40100000000000002, 0.27857773047946677], [-0.60787539226595488, 0.34100000000000003, 0.27163506072349475], [-0.8363954400004282, 0.20599999999999999, 0.2014662385347713], [-0.27576758055135975, 0.49399999999999999, 0.39136329273990866], [-0.17698658238285753, 0.38100000000000001, 0.42975946445365665], [-0.42301422073305678, 0.48599999999999999, 0.33614244280579275], [-0.81073978385996837, 0.161, 0.20875756070128504], [-0.63838151273605281, 0.107, 0.26161268117180692], [-0.64010006015388854, 0.0040000000000000001, 0.26105377495417703], [-0.62182425813185527, 0.22700000000000001, 0.2670287167605806], [2.6823577629697111, 0.014, 0.0036552616083728173], [-0.53272604021530889, 0.39400000000000002, 0.29711161877609882], [-0.15271881679589203, 0.34699999999999998, 0.43931001058875374], [-0.46563636027021654, 0.48499999999999999, 0.32073790602545205], [-0.67304141220538694, 0.32200000000000001, 0.25046047260054227], [-0.65348653890355268, 0.27000000000000002, 0.25672133238529804], [-0.042046555130192398, 0.36199999999999999, 0.48323079264157687], [-0.50234667391554988, 0.39400000000000002, 0.30771184147044006], [0.20134863894573138, 0.41199999999999998, 0.42021298643169991], [4.7562332747386709, 0.001, 9.8619270438859985e-07], [-0.17060076279397593, 0.44400000000000001, 0.43226884904800156], [2.7291423642735309, 0.02, 0.0031749642026865921], [-1.244478534224277, 0.036999999999999998, 0.10666174742715517], [2.6167445805079668, 0.045999999999999999, 0.0044386370442521805], [-0.81072889481557631, 0.20799999999999999, 0.20876068801239311], [-0.96834418270546307, 0.049000000000000002, 0.16643625291733932], [4.6354616023251367, 0.017000000000000001, 1.7807091640120731e-06], [1.711490849677441, 0.058999999999999997, 0.043495269461387376], [-0.68553064793121166, 0.317, 0.24650456386912545], [-0.10649304409706747, 0.44, 0.45759558701679381], [-0.54248639601227799, 0.29299999999999998, 0.29374173830183614], [0.031677897850072946, 0.311, 0.48736446049996618], [4.5356561799411086, 0.019, 2.8712332330593782e-06], [-1.3116013794941181, 0.01, 0.094827333672427794], [-1.3116013794941181, 0.0089999999999999993, 0.094827333672427794], [4.6393788826674198, 0.002, 1.7472893316661242e-06], [1.4382028801285995, 0.090999999999999998, 0.07518824913828126], [2.8328951008347012, 0.033000000000000002, 0.0023064260400579295], [-0.26084695643627254, 0.46500000000000002, 0.39710526578910033], [-0.78037465160516584, 0.122, 0.21758519184573832], [-0.64208684814893169, 0.307, 0.2604083973990674], [-1.3116013794941181, 0.001, 0.094827333672427794], [-0.57288206029351252, 0.314, 0.28336227604534636], [1.4616922569671136, 0.10000000000000001, 0.071912780017027833], [-0.10796036062141957, 0.38200000000000001, 0.45701356800520454], [0.25084361591996374, 0.29299999999999998, 0.40096750937264047], [-0.15481933481013774, 0.34699999999999998, 0.43848187401104366], [-0.15333330051666849, 0.35699999999999998, 0.43906772057781862], [-0.23163022692856874, 0.41699999999999998, 0.40841261410676866], [-0.34288722316108555, 0.28399999999999997, 0.36584165208609076], [-0.21867005782535454, 0.38200000000000001, 0.41345353745151792], [-0.79939347077393408, 0.215, 0.21203114760123465], [-0.57405515910861116, 0.19800000000000001, 0.28296523788747729], [0.17162418105442584, 0.151, 0.43186649789407894], [-0.45034522396490928, 0.435, 0.3262307674101308], [0.76919536546476719, 0.122, 0.22088867064513418], [0.43251102361572391, 0.14999999999999999, 0.33268502099692632], [0.37645002404345906, 0.155, 0.3532911812599], [-0.34984866752801247, 0.47699999999999998, 0.36322613637735446], [-0.48122747907298502, 0.439, 0.31517741642583419], [0.8295949731100305, 0.16700000000000001, 0.20338390993964683], [0.32735658947745261, 0.24399999999999999, 0.37169909789396838], [-0.30762465422828866, 0.47899999999999998, 0.37918397900139522], [0.50459229844619902, 0.14399999999999999, 0.30692261012051003], [0.7555990768924421, 0.14999999999999999, 0.22494480477952961]] diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index f56d5fb..137e8aa 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -14,6 +14,25 @@ import crankshaft.pysal_utils as pu from crankshaft import random_seeds import json +# Fixture files produced as follows +# +# import pysal as ps +# import numpy as np +# f = ps.open(ps.examples.get_path("stl_hom.txt")) +# y = np.array(f.by_col['HR8893']) +# w = ps.knnW_from_shapefile(ps.examples.get_path("stl_hom.shp"), k=5) +# +# out = [{"id": index, "neighbors": w.neighbors[index], "value": val} +# for index, val in enumerate(y)] +# with open('neighbors_getis.json', 'w') as f: +# f.write(str(out)) +# +# np.random.seed(1234) +# lgstar = ps.esda.getisord.G_Local(y, w, star=True, permutations=999) +# +# with open('getis_data.json', 'w') as f: +# f.write(str(zip(lgstar.z_sim, lgstar.p_sim, lgstar.p_z_sim))) + class GetisTest(unittest.TestCase): """Testing class for Getis-Ord's G funtion From 29de72de332d9381d3b1c4de078d58070887abcd Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 20 Sep 2016 09:55:13 -0400 Subject: [PATCH 24/48] output column renaming --- src/pg/sql/16_getis.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index 6e52d50..578f15a 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -9,8 +9,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 999, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') -RETURNS TABLE (z_val NUMERIC, p_val NUMERIC, p_z_sim NUMERIC, rowid BIGINT) +RETURNS TABLE (z_score NUMERIC, p_value NUMERIC, p_z_sim NUMERIC, rowid BIGINT) AS $$ from crankshaft.clustering import getis_ord return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; + +-- TODO: make a version that accepts the values as arrays From 166e9e223fac6de76e2d73dc143436d051f35216 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 20 Sep 2016 09:55:33 -0400 Subject: [PATCH 25/48] minor formatting changes --- src/pg/test/sql/16_getis_test.sql | 2 +- src/py/crankshaft/test/test_clustering_getis.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pg/test/sql/16_getis_test.sql b/src/pg/test/sql/16_getis_test.sql index 03cfcc6..2d01798 100644 --- a/src/pg/test/sql/16_getis_test.sql +++ b/src/pg/test/sql/16_getis_test.sql @@ -10,5 +10,5 @@ SELECT z_score, p_value FROM cdb_crankshaft.CDB_GetisOrdsG( 'select * from getis_data', 'hr8893', 'knn', 5, 999, - 'the_geom', 'cartodb_id') As cdb_getisordsg(z_score, p_value, p_z_sim) + 'the_geom', 'cartodb_id') As t(z_score, p_value, p_z_sim) WHERE p_value <= 0.05; diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 137e8aa..1bcc02d 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -28,14 +28,16 @@ import json # f.write(str(out)) # # np.random.seed(1234) +# # need to do random.seed(1234) too? # lgstar = ps.esda.getisord.G_Local(y, w, star=True, permutations=999) # -# with open('getis_data.json', 'w') as f: +# with open('getis.json', 'w') as f: # f.write(str(zip(lgstar.z_sim, lgstar.p_sim, lgstar.p_z_sim))) +# class GetisTest(unittest.TestCase): - """Testing class for Getis-Ord's G funtion + """Testing class for Getis-Ord's G* funtion This test replicates the work done in PySAL documentation: https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/autocorrelation.html#local-g-and-g """ From 258322fcca9f08b4999b71d62f5b451473f2af21 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 21 Sep 2016 11:46:07 -0400 Subject: [PATCH 26/48] update tests to queen weights from knn --- src/pg/test/expected/16_getis_test.out | 28 +- src/pg/test/sql/16_getis_test.sql | 11 +- src/py/crankshaft/test/fixtures/getis.json | 2 +- .../test/fixtures/neighbors_getis.json | 861 +----------------- .../crankshaft/test/test_clustering_getis.py | 37 +- 5 files changed, 45 insertions(+), 894 deletions(-) diff --git a/src/pg/test/expected/16_getis_test.out b/src/pg/test/expected/16_getis_test.out index ba5ee87..d0cb4f8 100644 --- a/src/pg/test/expected/16_getis_test.out +++ b/src/pg/test/expected/16_getis_test.out @@ -6,18 +6,16 @@ SET client_min_messages TO WARNING; _cdb_random_seeds (1 row) -z_score|p_value --1.0358|0.0160 --0.6404|0.0070 -2.6802|0.0190 -4.7529|0.0020 -2.7270|0.0250 --1.2444|0.0230 -4.6322|0.0150 -4.5325|0.0220 --1.3115|0.0090 --1.3115|0.0080 -4.6361|0.0010 -2.8307|0.0310 --1.3115|0.0020 -(13 rows) +rowid|z_score|p_value +9|-0.7862|0.0500 +22|-0.3955|0.0330 +33|2.7045|0.0050 +35|1.9524|0.0130 +36|-1.2056|0.0170 +37|3.4785|0.0020 +38|-1.4622|0.0020 +40|5.7098|0.0030 +46|3.4704|0.0120 +47|-0.9994|0.0320 +48|-1.3650|0.0340 +(11 rows) diff --git a/src/pg/test/sql/16_getis_test.sql b/src/pg/test/sql/16_getis_test.sql index 2d01798..90fb34d 100644 --- a/src/pg/test/sql/16_getis_test.sql +++ b/src/pg/test/sql/16_getis_test.sql @@ -5,10 +5,11 @@ -- set random seed SELECT cdb_crankshaft._cdb_random_seeds(1234); --- test against PySAL example dataset -SELECT z_score, p_value +-- test against PySAL example dataset 'stl_hom' +SELECT rowid, round(z_score, 4) As z_score, round(p_value, 4) As p_value FROM cdb_crankshaft.CDB_GetisOrdsG( 'select * from getis_data', - 'hr8893', 'knn', 5, 999, - 'the_geom', 'cartodb_id') As t(z_score, p_value, p_z_sim) -WHERE p_value <= 0.05; + 'hr8893', 'queen', NULL, 999, + 'the_geom', 'cartodb_id') As t(z_score, p_value, p_z_sim, rowid) +WHERE round(p_value, 4) <= 0.05 +ORDER BY rowid ASC; diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json index 4d5837f..ae67a3c 100644 --- a/src/py/crankshaft/test/fixtures/getis.json +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -1 +1 @@ -[[-0.37337341596824414, 0.43099999999999999, 0.35443527112672846], [-0.95217153199599269, 0.091999999999999998, 0.17050499804382691], [-0.69132084390888426, 0.30299999999999999, 0.24468196785399454], [-0.90994823039851658, 0.088999999999999996, 0.18142490623820451], [-1.0132361542216282, 0.129, 0.15547368761207747], [-0.51678782422627401, 0.058999999999999997, 0.30265214024759912], [-0.59887100659663139, 0.13400000000000001, 0.27462945349842682], [-0.65244852117509244, 0.32900000000000001, 0.2570559358979031], [-1.0357808174963727, 0.0070000000000000001, 0.1501522043425858], [-0.77003625124250874, 0.13600000000000001, 0.22063919456432224], [-0.33236541712916268, 0.375, 0.3698066743983055], [-0.16566702930134228, 0.28699999999999998, 0.43420949712949208], [-1.0188721243221961, 0.113, 0.15413183958086896], [-0.58707183737030266, 0.40100000000000002, 0.27857773047946677], [-0.60787539226595488, 0.34100000000000003, 0.27163506072349475], [-0.8363954400004282, 0.20599999999999999, 0.2014662385347713], [-0.27576758055135975, 0.49399999999999999, 0.39136329273990866], [-0.17698658238285753, 0.38100000000000001, 0.42975946445365665], [-0.42301422073305678, 0.48599999999999999, 0.33614244280579275], [-0.81073978385996837, 0.161, 0.20875756070128504], [-0.63838151273605281, 0.107, 0.26161268117180692], [-0.64010006015388854, 0.0040000000000000001, 0.26105377495417703], [-0.62182425813185527, 0.22700000000000001, 0.2670287167605806], [2.6823577629697111, 0.014, 0.0036552616083728173], [-0.53272604021530889, 0.39400000000000002, 0.29711161877609882], [-0.15271881679589203, 0.34699999999999998, 0.43931001058875374], [-0.46563636027021654, 0.48499999999999999, 0.32073790602545205], [-0.67304141220538694, 0.32200000000000001, 0.25046047260054227], [-0.65348653890355268, 0.27000000000000002, 0.25672133238529804], [-0.042046555130192398, 0.36199999999999999, 0.48323079264157687], [-0.50234667391554988, 0.39400000000000002, 0.30771184147044006], [0.20134863894573138, 0.41199999999999998, 0.42021298643169991], [4.7562332747386709, 0.001, 9.8619270438859985e-07], [-0.17060076279397593, 0.44400000000000001, 0.43226884904800156], [2.7291423642735309, 0.02, 0.0031749642026865921], [-1.244478534224277, 0.036999999999999998, 0.10666174742715517], [2.6167445805079668, 0.045999999999999999, 0.0044386370442521805], [-0.81072889481557631, 0.20799999999999999, 0.20876068801239311], [-0.96834418270546307, 0.049000000000000002, 0.16643625291733932], [4.6354616023251367, 0.017000000000000001, 1.7807091640120731e-06], [1.711490849677441, 0.058999999999999997, 0.043495269461387376], [-0.68553064793121166, 0.317, 0.24650456386912545], [-0.10649304409706747, 0.44, 0.45759558701679381], [-0.54248639601227799, 0.29299999999999998, 0.29374173830183614], [0.031677897850072946, 0.311, 0.48736446049996618], [4.5356561799411086, 0.019, 2.8712332330593782e-06], [-1.3116013794941181, 0.01, 0.094827333672427794], [-1.3116013794941181, 0.0089999999999999993, 0.094827333672427794], [4.6393788826674198, 0.002, 1.7472893316661242e-06], [1.4382028801285995, 0.090999999999999998, 0.07518824913828126], [2.8328951008347012, 0.033000000000000002, 0.0023064260400579295], [-0.26084695643627254, 0.46500000000000002, 0.39710526578910033], [-0.78037465160516584, 0.122, 0.21758519184573832], [-0.64208684814893169, 0.307, 0.2604083973990674], [-1.3116013794941181, 0.001, 0.094827333672427794], [-0.57288206029351252, 0.314, 0.28336227604534636], [1.4616922569671136, 0.10000000000000001, 0.071912780017027833], [-0.10796036062141957, 0.38200000000000001, 0.45701356800520454], [0.25084361591996374, 0.29299999999999998, 0.40096750937264047], [-0.15481933481013774, 0.34699999999999998, 0.43848187401104366], [-0.15333330051666849, 0.35699999999999998, 0.43906772057781862], [-0.23163022692856874, 0.41699999999999998, 0.40841261410676866], [-0.34288722316108555, 0.28399999999999997, 0.36584165208609076], [-0.21867005782535454, 0.38200000000000001, 0.41345353745151792], [-0.79939347077393408, 0.215, 0.21203114760123465], [-0.57405515910861116, 0.19800000000000001, 0.28296523788747729], [0.17162418105442584, 0.151, 0.43186649789407894], [-0.45034522396490928, 0.435, 0.3262307674101308], [0.76919536546476719, 0.122, 0.22088867064513418], [0.43251102361572391, 0.14999999999999999, 0.33268502099692632], [0.37645002404345906, 0.155, 0.3532911812599], [-0.34984866752801247, 0.47699999999999998, 0.36322613637735446], [-0.48122747907298502, 0.439, 0.31517741642583419], [0.8295949731100305, 0.16700000000000001, 0.20338390993964683], [0.32735658947745261, 0.24399999999999999, 0.37169909789396838], [-0.30762465422828866, 0.47899999999999998, 0.37918397900139522], [0.50459229844619902, 0.14399999999999999, 0.30692261012051003], [0.7555990768924421, 0.14999999999999999, 0.22494480477952961]] +[(0.004793783909323601, 0.17999999999999999, 0.49808756424021061), (-1.0701189472090842, 0.079000000000000001, 0.14228288580832316), (-0.67867750971877305, 0.42099999999999999, 0.24867110969448558), (-0.67407386707620487, 0.246, 0.25013217644612995), (-0.79495689068870035, 0.33200000000000002, 0.21331928959090596), (-0.49279481022182703, 0.058999999999999997, 0.31107878905057329), (-0.38075627530057132, 0.28399999999999997, 0.35169205342069643), (-0.86710921611314895, 0.23699999999999999, 0.19294108571294855), (-0.78618647240956485, 0.050000000000000003, 0.2158791250244505), (-0.76108527223116984, 0.064000000000000001, 0.22330306830813684), (-0.13340753531942209, 0.247, 0.44693554317763651), (-0.57584545722033043, 0.48999999999999999, 0.28235982246156488), (-0.78882694661192831, 0.433, 0.2151065788731219), (-0.38769767950046219, 0.375, 0.34911988661484239), (-0.56057819488052207, 0.41399999999999998, 0.28754255985169652), (-0.41354017495644935, 0.45500000000000002, 0.339605447117173), (-0.23993577722243081, 0.49099999999999999, 0.40519002230969337), (-0.1389080156677496, 0.40400000000000003, 0.44476141839645233), (-0.25485737510500855, 0.376, 0.39941662953554224), (-0.71218610582902353, 0.17399999999999999, 0.23817476979886087), (-0.54533105995872144, 0.13700000000000001, 0.2927629228714812), (-0.39547917847510977, 0.033000000000000002, 0.34624464252424236), (-0.43052658996257548, 0.35399999999999998, 0.33340631435564982), (-0.37296719193774736, 0.40300000000000002, 0.35458643102865428), (-0.66482612169465694, 0.31900000000000001, 0.25308085650392698), (-0.13772133540823422, 0.34699999999999998, 0.44523032843016275), (-0.6765304487868502, 0.20999999999999999, 0.24935196033890672), (-0.64518763494323472, 0.32200000000000001, 0.25940279912025543), (-0.5078622084312413, 0.41099999999999998, 0.30577498972600159), (-0.12652006733772059, 0.42899999999999999, 0.44966013262301163), (-0.32691133022814595, 0.498, 0.37186747562269029), (0.25533848511500978, 0.42399999999999999, 0.39923083899077472), (2.7045138116476508, 0.0050000000000000001, 0.0034202212972238577), (-0.1551614486076057, 0.44400000000000001, 0.43834701985429037), (1.9524487722567723, 0.012999999999999999, 0.025442473674991528), (-1.2055816465306763, 0.017000000000000001, 0.11398941970467646), (3.478472976017831, 0.002, 0.00025213964072468009), (-1.4621715757903719, 0.002, 0.071847099325659136), (-0.84010307600180256, 0.085000000000000006, 0.20042529779230778), (5.7097646237318243, 0.0030000000000000001, 5.6566262784940591e-09), (1.5082367956567375, 0.065000000000000002, 0.065746966514827365), (-0.58337270103430816, 0.44, 0.27982121546450034), (-0.083271860457022437, 0.45100000000000001, 0.46681768733385554), (-0.46872337815000953, 0.34599999999999997, 0.31963368715684204), (0.18490279849545319, 0.23799999999999999, 0.42665263797981101), (3.470424529947997, 0.012, 0.00025981817437825683), (-0.99942612137154796, 0.032000000000000001, 0.15879415560388499), (-1.3650387953594485, 0.034000000000000002, 0.08612042845912049), (1.8617160516432014, 0.081000000000000003, 0.03132156240215267), (1.1321188945775384, 0.11600000000000001, 0.12879222611766061), (0.064116686050580601, 0.27300000000000002, 0.4744386578180424), (-0.42032194540259099, 0.29999999999999999, 0.33712514016213468), (-0.79581215423980922, 0.123, 0.21307061309098785), (-0.42792753720906046, 0.45600000000000002, 0.33435193892883741), (-1.0629378527428395, 0.051999999999999998, 0.14390506780140866), (-0.54164761752225477, 0.33700000000000002, 0.29403064095211839), (1.0934778886820793, 0.13700000000000001, 0.13709201601893539), (-0.094068785378413719, 0.38200000000000001, 0.46252725802998929), (0.13482026574801856, 0.36799999999999999, 0.44637699118865737), (-0.13976995315653129, 0.34699999999999998, 0.44442087706276601), (-0.051047663924746682, 0.32000000000000001, 0.47964376985626245), (-0.21468297736730158, 0.41699999999999998, 0.41500724761906527), (-0.20873154637330626, 0.38800000000000001, 0.41732890604390893), (-0.32427876152583485, 0.49199999999999999, 0.37286349875557478), (-0.65254842943280977, 0.374, 0.25702372075306734), (-0.48611858196118796, 0.23300000000000001, 0.31344154643990074), (-0.14482354344529477, 0.32600000000000001, 0.44242509660469886), (-0.51052030974200002, 0.439, 0.30484349480873729), (0.56814382285283538, 0.14999999999999999, 0.28496865660103166), (0.58680919931668207, 0.161, 0.27866592887231878), (0.013390357044409013, 0.25800000000000001, 0.49465818005865647), (-0.19050728887961568, 0.41399999999999998, 0.4244558160399462), (-0.60531777422216049, 0.35199999999999998, 0.2724839368239631), (1.0899331115425805, 0.127, 0.13787130480311838), (0.17015055382651084, 0.36899999999999999, 0.43244586845546418), (-0.21738337124409801, 0.40600000000000003, 0.41395479459421991), (1.0329303331079593, 0.079000000000000001, 0.15081825117169467), (1.0218317101096221, 0.104, 0.15343027913308094)] \ No newline at end of file diff --git a/src/py/crankshaft/test/fixtures/neighbors_getis.json b/src/py/crankshaft/test/fixtures/neighbors_getis.json index 878b955..5c162d7 100644 --- a/src/py/crankshaft/test/fixtures/neighbors_getis.json +++ b/src/py/crankshaft/test/fixtures/neighbors_getis.json @@ -1,860 +1 @@ -[ - { - "neighbors": [ - 2, - 6, - 5, - 10, - 3 - ], - "id": 0, - "value": 1.624458 - }, - { - "neighbors": [ - 4, - 7, - 9, - 14, - 16 - ], - "id": 1, - "value": 2.255492 - }, - { - "neighbors": [ - 6, - 0, - 3, - 8, - 10 - ], - "id": 2, - "value": 1.46789 - }, - { - "neighbors": [ - 8, - 12, - 2, - 4, - 6 - ], - "id": 3, - "value": 2.484256 - }, - { - "neighbors": [ - 9, - 12, - 1, - 3, - 8 - ], - "id": 4, - "value": 0 - }, - { - "neighbors": [ - 11, - 10, - 0, - 15, - 6 - ], - "id": 5, - "value": 9.048673 - }, - { - "neighbors": [ - 2, - 10, - 0, - 8, - 17 - ], - "id": 6, - "value": 6.029489 - }, - { - "neighbors": [ - 14, - 1, - 22, - 16, - 9 - ], - "id": 7, - "value": 1.800385 - }, - { - "neighbors": [ - 3, - 12, - 19, - 2, - 18 - ], - "id": 8, - "value": 4.581251 - }, - { - "neighbors": [ - 4, - 16, - 12, - 1, - 20 - ], - "id": 9, - "value": 3.790607 - }, - { - "neighbors": [ - 17, - 6, - 5, - 15, - 0 - ], - "id": 10, - "value": 1.447436 - }, - { - "neighbors": [ - 15, - 5, - 13, - 21, - 27 - ], - "id": 11, - "value": 1.191966 - }, - { - "neighbors": [ - 8, - 19, - 3, - 9, - 4 - ], - "id": 12, - "value": 0 - }, - { - "neighbors": [ - 21, - 11, - 28, - 27, - 15 - ], - "id": 13, - "value": 1.608017 - }, - { - "neighbors": [ - 7, - 16, - 22, - 1, - 29 - ], - "id": 14, - "value": 1.949812 - }, - { - "neighbors": [ - 11, - 27, - 26, - 5, - 10 - ], - "id": 15, - "value": 0.74509 - }, - { - "neighbors": [ - 25, - 9, - 14, - 29, - 20 - ], - "id": 16, - "value": 4.173318 - }, - { - "neighbors": [ - 31, - 10, - 18, - 26, - 6 - ], - "id": 17, - "value": 3.783252 - }, - { - "neighbors": [ - 32, - 17, - 23, - 19, - 8 - ], - "id": 18, - "value": 2.085136 - }, - { - "neighbors": [ - 23, - 12, - 20, - 8, - 18 - ], - "id": 19, - "value": 2.176302 - }, - { - "neighbors": [ - 25, - 23, - 19, - 34, - 9 - ], - "id": 20, - "value": 6.309347 - }, - { - "neighbors": [ - 13, - 28, - 27, - 11, - 35 - ], - "id": 21, - "value": 10.855743 - }, - { - "neighbors": [ - 30, - 14, - 29, - 24, - 7 - ], - "id": 22, - "value": 4.211354 - }, - { - "neighbors": [ - 19, - 20, - 34, - 39, - 36 - ], - "id": 23, - "value": 0.80481 - }, - { - "neighbors": [ - 30, - 41, - 22, - 43, - 52 - ], - "id": 24, - "value": 3.215331 - }, - { - "neighbors": [ - 20, - 33, - 16, - 34, - 29 - ], - "id": 25, - "value": 2.833664 - }, - { - "neighbors": [ - 38, - 31, - 27, - 15, - 17 - ], - "id": 26, - "value": 1.59204 - }, - { - "neighbors": [ - 35, - 15, - 21, - 28, - 26 - ], - "id": 27, - "value": 1.571158 - }, - { - "neighbors": [ - 21, - 37, - 35, - 27, - 13 - ], - "id": 28, - "value": 3.12759 - }, - { - "neighbors": [ - 33, - 22, - 30, - 42, - 16 - ], - "id": 29, - "value": 4.416896 - }, - { - "neighbors": [ - 43, - 22, - 24, - 29, - 41 - ], - "id": 30, - "value": 3.017486 - }, - { - "neighbors": [ - 40, - 17, - 26, - 32, - 49 - ], - "id": 31, - "value": 9.924245 - }, - { - "neighbors": [ - 45, - 39, - 18, - 31, - 23 - ], - "id": 32, - "value": 7.973957 - }, - { - "neighbors": [ - 25, - 29, - 44, - 42, - 34 - ], - "id": 33, - "value": 5.005464 - }, - { - "neighbors": [ - 36, - 20, - 25, - 23, - 39 - ], - "id": 34, - "value": 2.463891 - }, - { - "neighbors": [ - 27, - 46, - 37, - 28, - 38 - ], - "id": 35, - "value": 0 - }, - { - "neighbors": [ - 39, - 34, - 50, - 48, - 23 - ], - "id": 36, - "value": 7.377974 - }, - { - "neighbors": [ - 47, - 28, - 35, - 46, - 21 - ], - "id": 37, - "value": 1.003875 - }, - { - "neighbors": [ - 51, - 26, - 35, - 40, - 27 - ], - "id": 38, - "value": 3.190047 - }, - { - "neighbors": [ - 36, - 45, - 48, - 32, - 23 - ], - "id": 39, - "value": 45.905406 - }, - { - "neighbors": [ - 49, - 31, - 38, - 45, - 57 - ], - "id": 40, - "value": 2.447597 - }, - { - "neighbors": [ - 52, - 43, - 30, - 24, - 53 - ], - "id": 41, - "value": 1.294958 - }, - { - "neighbors": [ - 43, - 44, - 33, - 53, - 29 - ], - "id": 42, - "value": 5.933098 - }, - { - "neighbors": [ - 53, - 42, - 30, - 41, - 60 - ], - "id": 43, - "value": 4.133997 - }, - { - "neighbors": [ - 33, - 42, - 59, - 58, - 34 - ], - "id": 44, - "value": 4.298311 - }, - { - "neighbors": [ - 48, - 39, - 32, - 56, - 40 - ], - "id": 45, - "value": 27.483827 - }, - { - "neighbors": [ - 35, - 55, - 47, - 54, - 37 - ], - "id": 46, - "value": 0.969791 - }, - { - "neighbors": [ - 37, - 54, - 46, - 35, - 55 - ], - "id": 47, - "value": 0 - }, - { - "neighbors": [ - 45, - 50, - 39, - 62, - 56 - ], - "id": 48, - "value": 2.934466 - }, - { - "neighbors": [ - 40, - 57, - 51, - 56, - 45 - ], - "id": 49, - "value": 4.456427 - }, - { - "neighbors": [ - 48, - 36, - 63, - 59, - 39 - ], - "id": 50, - "value": 4.629264 - }, - { - "neighbors": [ - 61, - 38, - 55, - 49, - 57 - ], - "id": 51, - "value": 4.941533 - }, - { - "neighbors": [ - 41, - 64, - 53, - 43, - 60 - ], - "id": 52, - "value": 3.990041 - }, - { - "neighbors": [ - 43, - 60, - 64, - 41, - 42 - ], - "id": 53, - "value": 2.064324 - }, - { - "neighbors": [ - 47, - 55, - 46, - 37, - 35 - ], - "id": 54, - "value": 3.040253 - }, - { - "neighbors": [ - 54, - 46, - 61, - 51, - 67 - ], - "id": 55, - "value": 3.905411 - }, - { - "neighbors": [ - 66, - 62, - 48, - 45, - 57 - ], - "id": 56, - "value": 4.332839 - }, - { - "neighbors": [ - 49, - 65, - 61, - 56, - 51 - ], - "id": 57, - "value": 3.894111 - }, - { - "neighbors": [ - 68, - 59, - 60, - 44, - 42 - ], - "id": 58, - "value": 6.828794 - }, - { - "neighbors": [ - 69, - 58, - 63, - 50, - 44 - ], - "id": 59, - "value": 3.263947 - }, - { - "neighbors": [ - 53, - 68, - 64, - 58, - 43 - ], - "id": 60, - "value": 3.282163 - }, - { - "neighbors": [ - 67, - 51, - 55, - 57, - 65 - ], - "id": 61, - "value": 3.295762 - }, - { - "neighbors": [ - 63, - 48, - 56, - 66, - 70 - ], - "id": 62, - "value": 7.249679 - }, - { - "neighbors": [ - 62, - 70, - 69, - 59, - 50 - ], - "id": 63, - "value": 3.041846 - }, - { - "neighbors": [ - 60, - 53, - 52, - 71, - 41 - ], - "id": 64, - "value": 1.618018 - }, - { - "neighbors": [ - 57, - 72, - 66, - 67, - 75 - ], - "id": 65, - "value": 4.910801 - }, - { - "neighbors": [ - 56, - 75, - 62, - 74, - 65 - ], - "id": 66, - "value": 1.991457 - }, - { - "neighbors": [ - 61, - 72, - 65, - 55, - 57 - ], - "id": 67, - "value": 3.146192 - }, - { - "neighbors": [ - 60, - 58, - 76, - 71, - 73 - ], - "id": 68, - "value": 7.26665 - }, - { - "neighbors": [ - 73, - 63, - 59, - 70, - 77 - ], - "id": 69, - "value": 3.110904 - }, - { - "neighbors": [ - 74, - 63, - 77, - 69, - 62 - ], - "id": 70, - "value": 2.980271 - }, - { - "neighbors": [ - 68, - 64, - 76, - 60, - 53 - ], - "id": 71, - "value": 3.866767 - }, - { - "neighbors": [ - 65, - 67, - 75, - 61, - 57 - ], - "id": 72, - "value": 1.868408 - }, - { - "neighbors": [ - 69, - 76, - 77, - 68, - 59 - ], - "id": 73, - "value": 12.577034 - }, - { - "neighbors": [ - 75, - 70, - 66, - 77, - 62 - ], - "id": 74, - "value": 7.803599 - }, - { - "neighbors": [ - 74, - 66, - 72, - 65, - 70 - ], - "id": 75, - "value": 3.47149 - }, - { - "neighbors": [ - 68, - 73, - 71, - 69, - 60 - ], - "id": 76, - "value": 4.334822 - }, - { - "neighbors": [ - 70, - 74, - 69, - 73, - 63 - ], - "id": 77, - "value": 8.451537 - } -] +[{'neighbors': [3, 6, 7], 'id': 1, 'value': 1.624458}, {'neighbors': [10, 5, 8], 'id': 2, 'value': 2.2554919999999998}, {'neighbors': [1, 4, 7], 'id': 3, 'value': 1.4678899999999999}, {'neighbors': [9, 3, 5, 7], 'id': 4, 'value': 2.4842559999999998}, {'neighbors': [9, 2, 4, 10], 'id': 5, 'value': 0.0}, {'neighbors': [1, 11, 12, 7, 16], 'id': 6, 'value': 9.0486730000000009}, {'neighbors': [1, 3, 4, 6, 9, 11, 18, 19], 'id': 7, 'value': 6.0294889999999999}, {'neighbors': [2, 15, 10], 'id': 8, 'value': 1.8003849999999999}, {'neighbors': [4, 5, 7, 10, 13, 19, 20], 'id': 9, 'value': 4.581251}, {'neighbors': [2, 5, 8, 9, 13, 15, 17, 20, 21], 'id': 10, 'value': 3.7906070000000001}, {'neighbors': [18, 6, 7, 16], 'id': 11, 'value': 1.4474359999999999}, {'neighbors': [16, 6, 14], 'id': 12, 'value': 1.1919660000000001}, {'neighbors': [9, 10, 20], 'id': 13, 'value': 0.0}, {'neighbors': [12, 22, 16], 'id': 14, 'value': 1.608017}, {'neighbors': [17, 10, 23, 8], 'id': 15, 'value': 1.9498120000000001}, {'neighbors': [6, 11, 12, 14, 18, 22, 27, 28], 'id': 16, 'value': 0.74509000000000003}, {'neighbors': [10, 15, 21, 23, 26, 30], 'id': 17, 'value': 4.1733180000000001}, {'neighbors': [33, 7, 11, 16, 19, 27, 32], 'id': 18, 'value': 3.7832520000000001}, {'neighbors': [33, 7, 9, 18, 20, 24], 'id': 19, 'value': 2.0851359999999999}, {'neighbors': [9, 10, 13, 19, 21, 24], 'id': 20, 'value': 2.1763020000000002}, {'neighbors': [35, 10, 17, 20, 24, 26], 'id': 21, 'value': 6.3093469999999998}, {'neighbors': [28, 29, 14, 16], 'id': 22, 'value': 10.855743}, {'neighbors': [17, 25, 31, 30, 15], 'id': 23, 'value': 4.211354}, {'neighbors': [33, 19, 20, 21, 35], 'id': 24, 'value': 0.80481000000000003}, {'neighbors': [42, 31, 23], 'id': 25, 'value': 3.2153309999999999}, {'neighbors': [17, 34, 35, 21, 30], 'id': 26, 'value': 2.8336640000000002}, {'neighbors': [36, 39, 41, 16, 18, 28, 32], 'id': 27, 'value': 1.5920399999999999}, {'neighbors': [27, 36, 29, 22, 16], 'id': 28, 'value': 1.5711580000000001}, {'neighbors': [36, 28, 22, 38], 'id': 29, 'value': 3.1275900000000001}, {'neighbors': [34, 43, 17, 23, 26, 31], 'id': 30, 'value': 4.4168960000000004}, {'neighbors': [42, 43, 44, 23, 25, 30], 'id': 31, 'value': 3.0174859999999999}, {'neighbors': [33, 18, 27, 41], 'id': 32, 'value': 9.9242450000000009}, {'neighbors': [35, 37, 40, 41, 46, 18, 19, 24, 32], 'id': 33, 'value': 7.9739570000000004}, {'neighbors': [26, 35, 43, 45, 30], 'id': 34, 'value': 5.0054639999999999}, {'neighbors': [33, 34, 37, 40, 45, 21, 24, 26], 'id': 35, 'value': 2.4638909999999998}, {'neighbors': [38, 39, 47, 27, 28, 29], 'id': 36, 'value': 0.0}, {'neighbors': [33, 35, 40, 45, 46, 49, 51], 'id': 37, 'value': 7.377974}, {'neighbors': [36, 29, 47, 48], 'id': 38, 'value': 1.0038750000000001}, {'neighbors': [36, 41, 47, 50, 52, 27], 'id': 39, 'value': 3.1900469999999999}, {'neighbors': [33, 35, 37, 46], 'id': 40, 'value': 45.905405999999999}, {'neighbors': [33, 39, 46, 50, 27, 32], 'id': 41, 'value': 2.447597}, {'neighbors': [25, 44, 53, 31], 'id': 42, 'value': 1.2949580000000001}, {'neighbors': [34, 44, 45, 54, 59, 61, 30, 31], 'id': 43, 'value': 5.9330980000000002}, {'neighbors': [42, 43, 53, 54, 31], 'id': 44, 'value': 4.1339969999999999}, {'neighbors': [34, 35, 37, 43, 51, 59, 60], 'id': 45, 'value': 4.298311}, {'neighbors': [33, 37, 40, 41, 49, 50, 57], 'id': 46, 'value': 27.483827000000002}, {'neighbors': [36, 38, 39, 48, 52, 55, 56], 'id': 47, 'value': 0.96979099999999996}, {'neighbors': [55, 38, 47], 'id': 48, 'value': 0.0}, {'neighbors': [57, 51, 37, 46, 63], 'id': 49, 'value': 2.934466}, {'neighbors': [39, 41, 46, 52, 57, 58], 'id': 50, 'value': 4.4564269999999997}, {'neighbors': [37, 45, 49, 60, 63, 64], 'id': 51, 'value': 4.629264}, {'neighbors': [39, 47, 50, 56, 58, 62], 'id': 52, 'value': 4.9415329999999997}, {'neighbors': [65, 42, 44, 54], 'id': 53, 'value': 3.9900410000000002}, {'neighbors': [65, 61, 43, 44, 53], 'id': 54, 'value': 2.064324}, {'neighbors': [56, 47, 48], 'id': 55, 'value': 3.0402529999999999}, {'neighbors': [52, 55, 47, 62], 'id': 56, 'value': 3.905411}, {'neighbors': [66, 67, 46, 49, 50, 58, 63], 'id': 57, 'value': 4.3328389999999999}, {'neighbors': [57, 50, 52, 62, 66], 'id': 58, 'value': 3.8941110000000001}, {'neighbors': [69, 70, 43, 45, 60, 61], 'id': 59, 'value': 6.8287940000000003}, {'neighbors': [51, 64, 45, 59, 70], 'id': 60, 'value': 3.2639469999999999}, {'neighbors': [65, 69, 72, 43, 54, 59], 'id': 61, 'value': 3.2821630000000002}, {'neighbors': [58, 68, 52, 66, 56], 'id': 62, 'value': 3.2957619999999999}, {'neighbors': [49, 57, 51, 67, 64], 'id': 63, 'value': 7.2496790000000004}, {'neighbors': [67, 70, 71, 51, 60, 63], 'id': 64, 'value': 3.041846}, {'neighbors': [61, 53, 54, 72], 'id': 65, 'value': 1.618018}, {'neighbors': [67, 68, 73, 76, 57, 58, 62], 'id': 66, 'value': 4.9108010000000002}, {'neighbors': [66, 71, 73, 75, 76, 57, 63, 64], 'id': 67, 'value': 1.991457}, {'neighbors': [73, 66, 62], 'id': 68, 'value': 3.1461920000000001}, {'neighbors': [70, 72, 74, 77, 59, 61], 'id': 69, 'value': 7.2666500000000003}, {'neighbors': [69, 71, 74, 78, 59, 60, 64], 'id': 70, 'value': 3.1109040000000001}, {'neighbors': [67, 75, 70, 78, 64], 'id': 71, 'value': 2.9802710000000001}, {'neighbors': [65, 69, 61, 77], 'id': 72, 'value': 3.8667669999999998}, {'neighbors': [76, 66, 67, 68], 'id': 73, 'value': 1.8684080000000001}, {'neighbors': [77, 69, 70, 78], 'id': 74, 'value': 12.577033999999999}, {'neighbors': [67, 76, 78, 71], 'id': 75, 'value': 7.8035990000000002}, {'neighbors': [73, 66, 67, 75], 'id': 76, 'value': 3.4714900000000002}, {'neighbors': [74, 69, 72], 'id': 77, 'value': 4.334822}, {'neighbors': [74, 75, 70, 71], 'id': 78, 'value': 8.4515370000000001}] \ No newline at end of file diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 1bcc02d..8acb562 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -18,22 +18,28 @@ import json # # import pysal as ps # import numpy as np -# f = ps.open(ps.examples.get_path("stl_hom.txt")) +# import random +# +# # setup variables +# f = ps.open(ps.examples.get_path("stl_hom.dbf")) # y = np.array(f.by_col['HR8893']) -# w = ps.knnW_from_shapefile(ps.examples.get_path("stl_hom.shp"), k=5) +# w_queen = ps.queen_from_shapefile(ps.examples.get_path("stl_hom.shp")) # -# out = [{"id": index, "neighbors": w.neighbors[index], "value": val} -# for index, val in enumerate(y)] -# with open('neighbors_getis.json', 'w') as f: -# f.write(str(out)) +# out_queen = [{"id": index + 1, +# "neighbors": [x+1 for x in w_queen.neighbors[index]], +# "value": val} for index, val in enumerate(y)] # +# with open('neighbors_queen_getis.json', 'w') as f: +# f.write(str(out_queen)) +# +# random.seed(1234) # np.random.seed(1234) -# # need to do random.seed(1234) too? -# lgstar = ps.esda.getisord.G_Local(y, w, star=True, permutations=999) -# -# with open('getis.json', 'w') as f: -# f.write(str(zip(lgstar.z_sim, lgstar.p_sim, lgstar.p_z_sim))) +# lgstar_queen = ps.esda.getisord.G_Local(y, w_queen, star=True, +# permutations=999) # +# with open('getis_queen.json', 'w') as f: +# f.write(str(zip(lgstar_queen.z_sim, +# lgstar_queen.p_sim, lgstar_queen.p_z_sim))) class GetisTest(unittest.TestCase): @@ -44,9 +50,14 @@ class GetisTest(unittest.TestCase): def setUp(self): plpy._reset() + + # load raw data for analysis self.neighbors_data = json.loads( open(fixture_file('neighbors_getis.json')).read()) - self.getis_data = json.loads(open(fixture_file('getis.json')).read()) + + # load pre-computed/known values + self.getis_data = json.loads( + open(fixture_file('getis.json')).read()) def test_getis_ord(self): """Test Getis-Ord's G*""" @@ -56,7 +67,7 @@ class GetisTest(unittest.TestCase): plpy._define_result('select', data) random_seeds.set_random_seeds(1234) result = cc.getis_ord('subquery', 'value', - 'knn', 5, 999, 'the_geom', 'cartodb_id') + 'queen', None, 999, 'the_geom', 'cartodb_id') result = [(row[0], row[1]) for row in result] expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): From e5ea83649327672d758a3b112f433f343543eb80 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 21 Sep 2016 11:53:17 -0400 Subject: [PATCH 27/48] fix json format --- src/py/crankshaft/test/fixtures/getis.json | 2 +- src/py/crankshaft/test/fixtures/neighbors_getis.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/test/fixtures/getis.json b/src/py/crankshaft/test/fixtures/getis.json index ae67a3c..02566fc 100644 --- a/src/py/crankshaft/test/fixtures/getis.json +++ b/src/py/crankshaft/test/fixtures/getis.json @@ -1 +1 @@ -[(0.004793783909323601, 0.17999999999999999, 0.49808756424021061), (-1.0701189472090842, 0.079000000000000001, 0.14228288580832316), (-0.67867750971877305, 0.42099999999999999, 0.24867110969448558), (-0.67407386707620487, 0.246, 0.25013217644612995), (-0.79495689068870035, 0.33200000000000002, 0.21331928959090596), (-0.49279481022182703, 0.058999999999999997, 0.31107878905057329), (-0.38075627530057132, 0.28399999999999997, 0.35169205342069643), (-0.86710921611314895, 0.23699999999999999, 0.19294108571294855), (-0.78618647240956485, 0.050000000000000003, 0.2158791250244505), (-0.76108527223116984, 0.064000000000000001, 0.22330306830813684), (-0.13340753531942209, 0.247, 0.44693554317763651), (-0.57584545722033043, 0.48999999999999999, 0.28235982246156488), (-0.78882694661192831, 0.433, 0.2151065788731219), (-0.38769767950046219, 0.375, 0.34911988661484239), (-0.56057819488052207, 0.41399999999999998, 0.28754255985169652), (-0.41354017495644935, 0.45500000000000002, 0.339605447117173), (-0.23993577722243081, 0.49099999999999999, 0.40519002230969337), (-0.1389080156677496, 0.40400000000000003, 0.44476141839645233), (-0.25485737510500855, 0.376, 0.39941662953554224), (-0.71218610582902353, 0.17399999999999999, 0.23817476979886087), (-0.54533105995872144, 0.13700000000000001, 0.2927629228714812), (-0.39547917847510977, 0.033000000000000002, 0.34624464252424236), (-0.43052658996257548, 0.35399999999999998, 0.33340631435564982), (-0.37296719193774736, 0.40300000000000002, 0.35458643102865428), (-0.66482612169465694, 0.31900000000000001, 0.25308085650392698), (-0.13772133540823422, 0.34699999999999998, 0.44523032843016275), (-0.6765304487868502, 0.20999999999999999, 0.24935196033890672), (-0.64518763494323472, 0.32200000000000001, 0.25940279912025543), (-0.5078622084312413, 0.41099999999999998, 0.30577498972600159), (-0.12652006733772059, 0.42899999999999999, 0.44966013262301163), (-0.32691133022814595, 0.498, 0.37186747562269029), (0.25533848511500978, 0.42399999999999999, 0.39923083899077472), (2.7045138116476508, 0.0050000000000000001, 0.0034202212972238577), (-0.1551614486076057, 0.44400000000000001, 0.43834701985429037), (1.9524487722567723, 0.012999999999999999, 0.025442473674991528), (-1.2055816465306763, 0.017000000000000001, 0.11398941970467646), (3.478472976017831, 0.002, 0.00025213964072468009), (-1.4621715757903719, 0.002, 0.071847099325659136), (-0.84010307600180256, 0.085000000000000006, 0.20042529779230778), (5.7097646237318243, 0.0030000000000000001, 5.6566262784940591e-09), (1.5082367956567375, 0.065000000000000002, 0.065746966514827365), (-0.58337270103430816, 0.44, 0.27982121546450034), (-0.083271860457022437, 0.45100000000000001, 0.46681768733385554), (-0.46872337815000953, 0.34599999999999997, 0.31963368715684204), (0.18490279849545319, 0.23799999999999999, 0.42665263797981101), (3.470424529947997, 0.012, 0.00025981817437825683), (-0.99942612137154796, 0.032000000000000001, 0.15879415560388499), (-1.3650387953594485, 0.034000000000000002, 0.08612042845912049), (1.8617160516432014, 0.081000000000000003, 0.03132156240215267), (1.1321188945775384, 0.11600000000000001, 0.12879222611766061), (0.064116686050580601, 0.27300000000000002, 0.4744386578180424), (-0.42032194540259099, 0.29999999999999999, 0.33712514016213468), (-0.79581215423980922, 0.123, 0.21307061309098785), (-0.42792753720906046, 0.45600000000000002, 0.33435193892883741), (-1.0629378527428395, 0.051999999999999998, 0.14390506780140866), (-0.54164761752225477, 0.33700000000000002, 0.29403064095211839), (1.0934778886820793, 0.13700000000000001, 0.13709201601893539), (-0.094068785378413719, 0.38200000000000001, 0.46252725802998929), (0.13482026574801856, 0.36799999999999999, 0.44637699118865737), (-0.13976995315653129, 0.34699999999999998, 0.44442087706276601), (-0.051047663924746682, 0.32000000000000001, 0.47964376985626245), (-0.21468297736730158, 0.41699999999999998, 0.41500724761906527), (-0.20873154637330626, 0.38800000000000001, 0.41732890604390893), (-0.32427876152583485, 0.49199999999999999, 0.37286349875557478), (-0.65254842943280977, 0.374, 0.25702372075306734), (-0.48611858196118796, 0.23300000000000001, 0.31344154643990074), (-0.14482354344529477, 0.32600000000000001, 0.44242509660469886), (-0.51052030974200002, 0.439, 0.30484349480873729), (0.56814382285283538, 0.14999999999999999, 0.28496865660103166), (0.58680919931668207, 0.161, 0.27866592887231878), (0.013390357044409013, 0.25800000000000001, 0.49465818005865647), (-0.19050728887961568, 0.41399999999999998, 0.4244558160399462), (-0.60531777422216049, 0.35199999999999998, 0.2724839368239631), (1.0899331115425805, 0.127, 0.13787130480311838), (0.17015055382651084, 0.36899999999999999, 0.43244586845546418), (-0.21738337124409801, 0.40600000000000003, 0.41395479459421991), (1.0329303331079593, 0.079000000000000001, 0.15081825117169467), (1.0218317101096221, 0.104, 0.15343027913308094)] \ No newline at end of file +[[0.004793783909323601, 0.17999999999999999, 0.49808756424021061], [-1.0701189472090842, 0.079000000000000001, 0.14228288580832316], [-0.67867750971877305, 0.42099999999999999, 0.24867110969448558], [-0.67407386707620487, 0.246, 0.25013217644612995], [-0.79495689068870035, 0.33200000000000002, 0.21331928959090596], [-0.49279481022182703, 0.058999999999999997, 0.31107878905057329], [-0.38075627530057132, 0.28399999999999997, 0.35169205342069643], [-0.86710921611314895, 0.23699999999999999, 0.19294108571294855], [-0.78618647240956485, 0.050000000000000003, 0.2158791250244505], [-0.76108527223116984, 0.064000000000000001, 0.22330306830813684], [-0.13340753531942209, 0.247, 0.44693554317763651], [-0.57584545722033043, 0.48999999999999999, 0.28235982246156488], [-0.78882694661192831, 0.433, 0.2151065788731219], [-0.38769767950046219, 0.375, 0.34911988661484239], [-0.56057819488052207, 0.41399999999999998, 0.28754255985169652], [-0.41354017495644935, 0.45500000000000002, 0.339605447117173], [-0.23993577722243081, 0.49099999999999999, 0.40519002230969337], [-0.1389080156677496, 0.40400000000000003, 0.44476141839645233], [-0.25485737510500855, 0.376, 0.39941662953554224], [-0.71218610582902353, 0.17399999999999999, 0.23817476979886087], [-0.54533105995872144, 0.13700000000000001, 0.2927629228714812], [-0.39547917847510977, 0.033000000000000002, 0.34624464252424236], [-0.43052658996257548, 0.35399999999999998, 0.33340631435564982], [-0.37296719193774736, 0.40300000000000002, 0.35458643102865428], [-0.66482612169465694, 0.31900000000000001, 0.25308085650392698], [-0.13772133540823422, 0.34699999999999998, 0.44523032843016275], [-0.6765304487868502, 0.20999999999999999, 0.24935196033890672], [-0.64518763494323472, 0.32200000000000001, 0.25940279912025543], [-0.5078622084312413, 0.41099999999999998, 0.30577498972600159], [-0.12652006733772059, 0.42899999999999999, 0.44966013262301163], [-0.32691133022814595, 0.498, 0.37186747562269029], [0.25533848511500978, 0.42399999999999999, 0.39923083899077472], [2.7045138116476508, 0.0050000000000000001, 0.0034202212972238577], [-0.1551614486076057, 0.44400000000000001, 0.43834701985429037], [1.9524487722567723, 0.012999999999999999, 0.025442473674991528], [-1.2055816465306763, 0.017000000000000001, 0.11398941970467646], [3.478472976017831, 0.002, 0.00025213964072468009], [-1.4621715757903719, 0.002, 0.071847099325659136], [-0.84010307600180256, 0.085000000000000006, 0.20042529779230778], [5.7097646237318243, 0.0030000000000000001, 5.6566262784940591e-09], [1.5082367956567375, 0.065000000000000002, 0.065746966514827365], [-0.58337270103430816, 0.44, 0.27982121546450034], [-0.083271860457022437, 0.45100000000000001, 0.46681768733385554], [-0.46872337815000953, 0.34599999999999997, 0.31963368715684204], [0.18490279849545319, 0.23799999999999999, 0.42665263797981101], [3.470424529947997, 0.012, 0.00025981817437825683], [-0.99942612137154796, 0.032000000000000001, 0.15879415560388499], [-1.3650387953594485, 0.034000000000000002, 0.08612042845912049], [1.8617160516432014, 0.081000000000000003, 0.03132156240215267], [1.1321188945775384, 0.11600000000000001, 0.12879222611766061], [0.064116686050580601, 0.27300000000000002, 0.4744386578180424], [-0.42032194540259099, 0.29999999999999999, 0.33712514016213468], [-0.79581215423980922, 0.123, 0.21307061309098785], [-0.42792753720906046, 0.45600000000000002, 0.33435193892883741], [-1.0629378527428395, 0.051999999999999998, 0.14390506780140866], [-0.54164761752225477, 0.33700000000000002, 0.29403064095211839], [1.0934778886820793, 0.13700000000000001, 0.13709201601893539], [-0.094068785378413719, 0.38200000000000001, 0.46252725802998929], [0.13482026574801856, 0.36799999999999999, 0.44637699118865737], [-0.13976995315653129, 0.34699999999999998, 0.44442087706276601], [-0.051047663924746682, 0.32000000000000001, 0.47964376985626245], [-0.21468297736730158, 0.41699999999999998, 0.41500724761906527], [-0.20873154637330626, 0.38800000000000001, 0.41732890604390893], [-0.32427876152583485, 0.49199999999999999, 0.37286349875557478], [-0.65254842943280977, 0.374, 0.25702372075306734], [-0.48611858196118796, 0.23300000000000001, 0.31344154643990074], [-0.14482354344529477, 0.32600000000000001, 0.44242509660469886], [-0.51052030974200002, 0.439, 0.30484349480873729], [0.56814382285283538, 0.14999999999999999, 0.28496865660103166], [0.58680919931668207, 0.161, 0.27866592887231878], [0.013390357044409013, 0.25800000000000001, 0.49465818005865647], [-0.19050728887961568, 0.41399999999999998, 0.4244558160399462], [-0.60531777422216049, 0.35199999999999998, 0.2724839368239631], [1.0899331115425805, 0.127, 0.13787130480311838], [0.17015055382651084, 0.36899999999999999, 0.43244586845546418], [-0.21738337124409801, 0.40600000000000003, 0.41395479459421991], [1.0329303331079593, 0.079000000000000001, 0.15081825117169467], [1.0218317101096221, 0.104, 0.15343027913308094]] diff --git a/src/py/crankshaft/test/fixtures/neighbors_getis.json b/src/py/crankshaft/test/fixtures/neighbors_getis.json index 5c162d7..be367ff 100644 --- a/src/py/crankshaft/test/fixtures/neighbors_getis.json +++ b/src/py/crankshaft/test/fixtures/neighbors_getis.json @@ -1 +1 @@ -[{'neighbors': [3, 6, 7], 'id': 1, 'value': 1.624458}, {'neighbors': [10, 5, 8], 'id': 2, 'value': 2.2554919999999998}, {'neighbors': [1, 4, 7], 'id': 3, 'value': 1.4678899999999999}, {'neighbors': [9, 3, 5, 7], 'id': 4, 'value': 2.4842559999999998}, {'neighbors': [9, 2, 4, 10], 'id': 5, 'value': 0.0}, {'neighbors': [1, 11, 12, 7, 16], 'id': 6, 'value': 9.0486730000000009}, {'neighbors': [1, 3, 4, 6, 9, 11, 18, 19], 'id': 7, 'value': 6.0294889999999999}, {'neighbors': [2, 15, 10], 'id': 8, 'value': 1.8003849999999999}, {'neighbors': [4, 5, 7, 10, 13, 19, 20], 'id': 9, 'value': 4.581251}, {'neighbors': [2, 5, 8, 9, 13, 15, 17, 20, 21], 'id': 10, 'value': 3.7906070000000001}, {'neighbors': [18, 6, 7, 16], 'id': 11, 'value': 1.4474359999999999}, {'neighbors': [16, 6, 14], 'id': 12, 'value': 1.1919660000000001}, {'neighbors': [9, 10, 20], 'id': 13, 'value': 0.0}, {'neighbors': [12, 22, 16], 'id': 14, 'value': 1.608017}, {'neighbors': [17, 10, 23, 8], 'id': 15, 'value': 1.9498120000000001}, {'neighbors': [6, 11, 12, 14, 18, 22, 27, 28], 'id': 16, 'value': 0.74509000000000003}, {'neighbors': [10, 15, 21, 23, 26, 30], 'id': 17, 'value': 4.1733180000000001}, {'neighbors': [33, 7, 11, 16, 19, 27, 32], 'id': 18, 'value': 3.7832520000000001}, {'neighbors': [33, 7, 9, 18, 20, 24], 'id': 19, 'value': 2.0851359999999999}, {'neighbors': [9, 10, 13, 19, 21, 24], 'id': 20, 'value': 2.1763020000000002}, {'neighbors': [35, 10, 17, 20, 24, 26], 'id': 21, 'value': 6.3093469999999998}, {'neighbors': [28, 29, 14, 16], 'id': 22, 'value': 10.855743}, {'neighbors': [17, 25, 31, 30, 15], 'id': 23, 'value': 4.211354}, {'neighbors': [33, 19, 20, 21, 35], 'id': 24, 'value': 0.80481000000000003}, {'neighbors': [42, 31, 23], 'id': 25, 'value': 3.2153309999999999}, {'neighbors': [17, 34, 35, 21, 30], 'id': 26, 'value': 2.8336640000000002}, {'neighbors': [36, 39, 41, 16, 18, 28, 32], 'id': 27, 'value': 1.5920399999999999}, {'neighbors': [27, 36, 29, 22, 16], 'id': 28, 'value': 1.5711580000000001}, {'neighbors': [36, 28, 22, 38], 'id': 29, 'value': 3.1275900000000001}, {'neighbors': [34, 43, 17, 23, 26, 31], 'id': 30, 'value': 4.4168960000000004}, {'neighbors': [42, 43, 44, 23, 25, 30], 'id': 31, 'value': 3.0174859999999999}, {'neighbors': [33, 18, 27, 41], 'id': 32, 'value': 9.9242450000000009}, {'neighbors': [35, 37, 40, 41, 46, 18, 19, 24, 32], 'id': 33, 'value': 7.9739570000000004}, {'neighbors': [26, 35, 43, 45, 30], 'id': 34, 'value': 5.0054639999999999}, {'neighbors': [33, 34, 37, 40, 45, 21, 24, 26], 'id': 35, 'value': 2.4638909999999998}, {'neighbors': [38, 39, 47, 27, 28, 29], 'id': 36, 'value': 0.0}, {'neighbors': [33, 35, 40, 45, 46, 49, 51], 'id': 37, 'value': 7.377974}, {'neighbors': [36, 29, 47, 48], 'id': 38, 'value': 1.0038750000000001}, {'neighbors': [36, 41, 47, 50, 52, 27], 'id': 39, 'value': 3.1900469999999999}, {'neighbors': [33, 35, 37, 46], 'id': 40, 'value': 45.905405999999999}, {'neighbors': [33, 39, 46, 50, 27, 32], 'id': 41, 'value': 2.447597}, {'neighbors': [25, 44, 53, 31], 'id': 42, 'value': 1.2949580000000001}, {'neighbors': [34, 44, 45, 54, 59, 61, 30, 31], 'id': 43, 'value': 5.9330980000000002}, {'neighbors': [42, 43, 53, 54, 31], 'id': 44, 'value': 4.1339969999999999}, {'neighbors': [34, 35, 37, 43, 51, 59, 60], 'id': 45, 'value': 4.298311}, {'neighbors': [33, 37, 40, 41, 49, 50, 57], 'id': 46, 'value': 27.483827000000002}, {'neighbors': [36, 38, 39, 48, 52, 55, 56], 'id': 47, 'value': 0.96979099999999996}, {'neighbors': [55, 38, 47], 'id': 48, 'value': 0.0}, {'neighbors': [57, 51, 37, 46, 63], 'id': 49, 'value': 2.934466}, {'neighbors': [39, 41, 46, 52, 57, 58], 'id': 50, 'value': 4.4564269999999997}, {'neighbors': [37, 45, 49, 60, 63, 64], 'id': 51, 'value': 4.629264}, {'neighbors': [39, 47, 50, 56, 58, 62], 'id': 52, 'value': 4.9415329999999997}, {'neighbors': [65, 42, 44, 54], 'id': 53, 'value': 3.9900410000000002}, {'neighbors': [65, 61, 43, 44, 53], 'id': 54, 'value': 2.064324}, {'neighbors': [56, 47, 48], 'id': 55, 'value': 3.0402529999999999}, {'neighbors': [52, 55, 47, 62], 'id': 56, 'value': 3.905411}, {'neighbors': [66, 67, 46, 49, 50, 58, 63], 'id': 57, 'value': 4.3328389999999999}, {'neighbors': [57, 50, 52, 62, 66], 'id': 58, 'value': 3.8941110000000001}, {'neighbors': [69, 70, 43, 45, 60, 61], 'id': 59, 'value': 6.8287940000000003}, {'neighbors': [51, 64, 45, 59, 70], 'id': 60, 'value': 3.2639469999999999}, {'neighbors': [65, 69, 72, 43, 54, 59], 'id': 61, 'value': 3.2821630000000002}, {'neighbors': [58, 68, 52, 66, 56], 'id': 62, 'value': 3.2957619999999999}, {'neighbors': [49, 57, 51, 67, 64], 'id': 63, 'value': 7.2496790000000004}, {'neighbors': [67, 70, 71, 51, 60, 63], 'id': 64, 'value': 3.041846}, {'neighbors': [61, 53, 54, 72], 'id': 65, 'value': 1.618018}, {'neighbors': [67, 68, 73, 76, 57, 58, 62], 'id': 66, 'value': 4.9108010000000002}, {'neighbors': [66, 71, 73, 75, 76, 57, 63, 64], 'id': 67, 'value': 1.991457}, {'neighbors': [73, 66, 62], 'id': 68, 'value': 3.1461920000000001}, {'neighbors': [70, 72, 74, 77, 59, 61], 'id': 69, 'value': 7.2666500000000003}, {'neighbors': [69, 71, 74, 78, 59, 60, 64], 'id': 70, 'value': 3.1109040000000001}, {'neighbors': [67, 75, 70, 78, 64], 'id': 71, 'value': 2.9802710000000001}, {'neighbors': [65, 69, 61, 77], 'id': 72, 'value': 3.8667669999999998}, {'neighbors': [76, 66, 67, 68], 'id': 73, 'value': 1.8684080000000001}, {'neighbors': [77, 69, 70, 78], 'id': 74, 'value': 12.577033999999999}, {'neighbors': [67, 76, 78, 71], 'id': 75, 'value': 7.8035990000000002}, {'neighbors': [73, 66, 67, 75], 'id': 76, 'value': 3.4714900000000002}, {'neighbors': [74, 69, 72], 'id': 77, 'value': 4.334822}, {'neighbors': [74, 75, 70, 71], 'id': 78, 'value': 8.4515370000000001}] \ No newline at end of file +[{"neighbors": [3, 6, 7], "id": 1, "value": 1.624458}, {"neighbors": [10, 5, 8], "id": 2, "value": 2.2554919999999998}, {"neighbors": [1, 4, 7], "id": 3, "value": 1.4678899999999999}, {"neighbors": [9, 3, 5, 7], "id": 4, "value": 2.4842559999999998}, {"neighbors": [9, 2, 4, 10], "id": 5, "value": 0.0}, {"neighbors": [1, 11, 12, 7, 16], "id": 6, "value": 9.0486730000000009}, {"neighbors": [1, 3, 4, 6, 9, 11, 18, 19], "id": 7, "value": 6.0294889999999999}, {"neighbors": [2, 15, 10], "id": 8, "value": 1.8003849999999999}, {"neighbors": [4, 5, 7, 10, 13, 19, 20], "id": 9, "value": 4.581251}, {"neighbors": [2, 5, 8, 9, 13, 15, 17, 20, 21], "id": 10, "value": 3.7906070000000001}, {"neighbors": [18, 6, 7, 16], "id": 11, "value": 1.4474359999999999}, {"neighbors": [16, 6, 14], "id": 12, "value": 1.1919660000000001}, {"neighbors": [9, 10, 20], "id": 13, "value": 0.0}, {"neighbors": [12, 22, 16], "id": 14, "value": 1.608017}, {"neighbors": [17, 10, 23, 8], "id": 15, "value": 1.9498120000000001}, {"neighbors": [6, 11, 12, 14, 18, 22, 27, 28], "id": 16, "value": 0.74509000000000003}, {"neighbors": [10, 15, 21, 23, 26, 30], "id": 17, "value": 4.1733180000000001}, {"neighbors": [33, 7, 11, 16, 19, 27, 32], "id": 18, "value": 3.7832520000000001}, {"neighbors": [33, 7, 9, 18, 20, 24], "id": 19, "value": 2.0851359999999999}, {"neighbors": [9, 10, 13, 19, 21, 24], "id": 20, "value": 2.1763020000000002}, {"neighbors": [35, 10, 17, 20, 24, 26], "id": 21, "value": 6.3093469999999998}, {"neighbors": [28, 29, 14, 16], "id": 22, "value": 10.855743}, {"neighbors": [17, 25, 31, 30, 15], "id": 23, "value": 4.211354}, {"neighbors": [33, 19, 20, 21, 35], "id": 24, "value": 0.80481000000000003}, {"neighbors": [42, 31, 23], "id": 25, "value": 3.2153309999999999}, {"neighbors": [17, 34, 35, 21, 30], "id": 26, "value": 2.8336640000000002}, {"neighbors": [36, 39, 41, 16, 18, 28, 32], "id": 27, "value": 1.5920399999999999}, {"neighbors": [27, 36, 29, 22, 16], "id": 28, "value": 1.5711580000000001}, {"neighbors": [36, 28, 22, 38], "id": 29, "value": 3.1275900000000001}, {"neighbors": [34, 43, 17, 23, 26, 31], "id": 30, "value": 4.4168960000000004}, {"neighbors": [42, 43, 44, 23, 25, 30], "id": 31, "value": 3.0174859999999999}, {"neighbors": [33, 18, 27, 41], "id": 32, "value": 9.9242450000000009}, {"neighbors": [35, 37, 40, 41, 46, 18, 19, 24, 32], "id": 33, "value": 7.9739570000000004}, {"neighbors": [26, 35, 43, 45, 30], "id": 34, "value": 5.0054639999999999}, {"neighbors": [33, 34, 37, 40, 45, 21, 24, 26], "id": 35, "value": 2.4638909999999998}, {"neighbors": [38, 39, 47, 27, 28, 29], "id": 36, "value": 0.0}, {"neighbors": [33, 35, 40, 45, 46, 49, 51], "id": 37, "value": 7.377974}, {"neighbors": [36, 29, 47, 48], "id": 38, "value": 1.0038750000000001}, {"neighbors": [36, 41, 47, 50, 52, 27], "id": 39, "value": 3.1900469999999999}, {"neighbors": [33, 35, 37, 46], "id": 40, "value": 45.905405999999999}, {"neighbors": [33, 39, 46, 50, 27, 32], "id": 41, "value": 2.447597}, {"neighbors": [25, 44, 53, 31], "id": 42, "value": 1.2949580000000001}, {"neighbors": [34, 44, 45, 54, 59, 61, 30, 31], "id": 43, "value": 5.9330980000000002}, {"neighbors": [42, 43, 53, 54, 31], "id": 44, "value": 4.1339969999999999}, {"neighbors": [34, 35, 37, 43, 51, 59, 60], "id": 45, "value": 4.298311}, {"neighbors": [33, 37, 40, 41, 49, 50, 57], "id": 46, "value": 27.483827000000002}, {"neighbors": [36, 38, 39, 48, 52, 55, 56], "id": 47, "value": 0.96979099999999996}, {"neighbors": [55, 38, 47], "id": 48, "value": 0.0}, {"neighbors": [57, 51, 37, 46, 63], "id": 49, "value": 2.934466}, {"neighbors": [39, 41, 46, 52, 57, 58], "id": 50, "value": 4.4564269999999997}, {"neighbors": [37, 45, 49, 60, 63, 64], "id": 51, "value": 4.629264}, {"neighbors": [39, 47, 50, 56, 58, 62], "id": 52, "value": 4.9415329999999997}, {"neighbors": [65, 42, 44, 54], "id": 53, "value": 3.9900410000000002}, {"neighbors": [65, 61, 43, 44, 53], "id": 54, "value": 2.064324}, {"neighbors": [56, 47, 48], "id": 55, "value": 3.0402529999999999}, {"neighbors": [52, 55, 47, 62], "id": 56, "value": 3.905411}, {"neighbors": [66, 67, 46, 49, 50, 58, 63], "id": 57, "value": 4.3328389999999999}, {"neighbors": [57, 50, 52, 62, 66], "id": 58, "value": 3.8941110000000001}, {"neighbors": [69, 70, 43, 45, 60, 61], "id": 59, "value": 6.8287940000000003}, {"neighbors": [51, 64, 45, 59, 70], "id": 60, "value": 3.2639469999999999}, {"neighbors": [65, 69, 72, 43, 54, 59], "id": 61, "value": 3.2821630000000002}, {"neighbors": [58, 68, 52, 66, 56], "id": 62, "value": 3.2957619999999999}, {"neighbors": [49, 57, 51, 67, 64], "id": 63, "value": 7.2496790000000004}, {"neighbors": [67, 70, 71, 51, 60, 63], "id": 64, "value": 3.041846}, {"neighbors": [61, 53, 54, 72], "id": 65, "value": 1.618018}, {"neighbors": [67, 68, 73, 76, 57, 58, 62], "id": 66, "value": 4.9108010000000002}, {"neighbors": [66, 71, 73, 75, 76, 57, 63, 64], "id": 67, "value": 1.991457}, {"neighbors": [73, 66, 62], "id": 68, "value": 3.1461920000000001}, {"neighbors": [70, 72, 74, 77, 59, 61], "id": 69, "value": 7.2666500000000003}, {"neighbors": [69, 71, 74, 78, 59, 60, 64], "id": 70, "value": 3.1109040000000001}, {"neighbors": [67, 75, 70, 78, 64], "id": 71, "value": 2.9802710000000001}, {"neighbors": [65, 69, 61, 77], "id": 72, "value": 3.8667669999999998}, {"neighbors": [76, 66, 67, 68], "id": 73, "value": 1.8684080000000001}, {"neighbors": [77, 69, 70, 78], "id": 74, "value": 12.577033999999999}, {"neighbors": [67, 76, 78, 71], "id": 75, "value": 7.8035990000000002}, {"neighbors": [73, 66, 67, 75], "id": 76, "value": 3.4714900000000002}, {"neighbors": [74, 69, 72], "id": 77, "value": 4.334822}, {"neighbors": [74, 75, 70, 71], "id": 78, "value": 8.4515370000000001}] From 795413e46dc77ab473a6a78fe97fd4b4749a1eb4 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 21 Sep 2016 12:01:42 -0400 Subject: [PATCH 28/48] cleaning test files --- src/pg/test/expected/16_getis_test.out | 2 +- src/pg/test/fixtures/getis_data.sql | 9 ++------- src/py/crankshaft/test/test_clustering_getis.py | 2 -- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/pg/test/expected/16_getis_test.out b/src/pg/test/expected/16_getis_test.out index d0cb4f8..416844b 100644 --- a/src/pg/test/expected/16_getis_test.out +++ b/src/pg/test/expected/16_getis_test.out @@ -1,6 +1,6 @@ \pset format unaligned \set ECHO all -\i test/fixtures/ppoints.sql +\i test/fixtures/getis_data.sql SET client_min_messages TO WARNING; \set ECHO none _cdb_random_seeds diff --git a/src/pg/test/fixtures/getis_data.sql b/src/pg/test/fixtures/getis_data.sql index e7b2f69..3b47013 100644 --- a/src/pg/test/fixtures/getis_data.sql +++ b/src/pg/test/fixtures/getis_data.sql @@ -3,20 +3,16 @@ SET client_min_messages TO WARNING; -- -- Getis-Ord's G* test dataset, subsetted from PySAL examples: --- https://github.com/pysal/pysal/tree/952ea04029165048a774d9a1846cf86ad000c096/pysal/examples/stl +-- https://github.com/pysal/pysal/tree/952ea04029165048a774d9a1846cf86ad000c096/pysal/examples/stl -- CREATE TABLE getis_data ( cartodb_id integer, the_geom geometry(Geometry,4326), - hr8893 numeric + hr8893 numeric ); - -ALTER TABLE getis_data OWNER TO contrib_regression; - - COPY getis_data (cartodb_id, the_geom, hr8893) FROM stdin; 22 0106000020E61000000100000001030000000100000007000000000000E0B10056C0000000C0B8964340FFFFFFFF4C1756C00000002054964340000000A00F1E56C00000004072964340000000C02D1E56C0000000A0439B434000000060381E56C00000000036B04340000000E0E20056C0000000608CB04340000000E0B10056C0000000C0B8964340 10.8557430000000004 32 0106000020E6100000010000000103000000010000000B000000FFFFFF1FC26656C0FFFFFFBFE25E4340000000A0D86656C0000000E0976F4340000000A03A6956C0000000C0966F434000000020526956C0000000E08A7F4340000000E0F26556C000000000C87F4340000000E0066656C0000000209C834340000000407F5056C0000000803C83434000000020635056C0000000E016814340000000A0F45056C0000000A0F980434000000060D25056C000000060FA5E4340FFFFFF1FC26656C0FFFFFFBFE25E4340 9.92424500000000087 @@ -100,4 +96,3 @@ COPY getis_data (cartodb_id, the_geom, hr8893) FROM stdin; CREATE INDEX getis_data_gix ON getis_data USING GIST(the_geom); - diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 8acb562..835a121 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -72,5 +72,3 @@ class GetisTest(unittest.TestCase): expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): self.assertAlmostEqual(res_z, exp_z, delta=1e-2) - if exp_p <= 0.05: - self.assertTrue(res_p < 0.05) From 5443b674707c92b77778fabd949051ce0c52857d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 22 Sep 2016 08:58:22 -0400 Subject: [PATCH 29/48] adding docs for getis ord's g --- doc/16_getis_ord_gstar.md | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 doc/16_getis_ord_gstar.md diff --git a/doc/16_getis_ord_gstar.md b/doc/16_getis_ord_gstar.md new file mode 100644 index 0000000..1b12ac9 --- /dev/null +++ b/doc/16_getis_ord_gstar.md @@ -0,0 +1,40 @@ +## Getis-Ord's G\* + +Getis-Ord's G\* is a geo-statistical measurement of the intensity of clustering of high or low values. The clustering of high values can be referred to as "hotspots" because these are areas of high activity or large (relative to the global mean) measurement values. Coldspots are clustered areas with low activity or small measurement values. + +### CDB_GetisOrdsG(subquery text, column_name text) + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | text | A query of the data you want to pass to the function. It must include `column_name`, a geometry column (usually `the_geom`) and an id column (usually `cartodb_id`) | +| column_name | text | This is the column of interest for performing this analysis on. This column should be a numeric type. | +| w_type (optional) | text | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation.](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html) | +| num_ngbrs (optional) | integer | Default: 5. If `knn` is chosen, this will set the number of neighbors. If `knn` is not chosen, any entered value will be ignored. Use `NULL` if not choosing `knn`. | +| permutations (optional) | integer | The number of permutations for calculating p-values. Default: 999 | +| geom_col (optional) | text | The column where the geometry information is stored. The format must be PostGIS Geometry type (SRID 4326). Default: `the_geom`. | +| id_col (optional) | text | The column that has the unique row identifier. | + +### Returns + +Returns a table with the following columns. + +| Name | Type | Description | +|------|------|-------------| +| z_score | numeric | z-score, a measure of the intensity of clustering of high values (hotspots) or low values (coldspots). Positive values represent 'hotspots', while negative values represent 'coldspots'. | +| p_value | numeric | p-value, a measure of the significance of the intensity of clustering | +| p_z_sim | numeric | p-value based on standard normal approximation from permutations | +| rowid | integer | The original `id_col` that can be used to associate the outputs with the original geometry and inputs | + +#### Example Usage + +The following query returns the original table augmented with the values calculated from the Getis-Ord's G\* analysis. + +```sql +SELECT i.*, m.z_score, m.p_value + FROM cdb_crankshaft.CDB_GetisOrdsG('SELECT * FROM incident_reports_clustered', + 'num_incidents') As m + JOIN incident_reports_clustered As i + ON i.cartodb_id = m.rowid; +``` From 06452562b905f21dc0adb6a0fa403cda333bb737 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 26 Sep 2016 10:10:52 -0400 Subject: [PATCH 30/48] fix ordering problems in input columns --- src/py/crankshaft/test/test_pysal_utils.py | 81 ++++++++++------------ 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index aad9b20..92b528b 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -9,23 +9,23 @@ class PysalUtilsTest(unittest.TestCase): """Testing class for utility functions related to PySAL integrations""" def setUp(self): - self.params = OrderedDict([("id_col", "cartodb_id"), - ("attr1", "andy"), - ("attr2", "jay_z"), - ("subquery", "SELECT * FROM a_list"), - ("geom_col", "the_geom"), - ("num_ngbrs", 321)]) - - self.params2 = OrderedDict([("id_col", "cartodb_id"), - ("attr1", "price"), - ("attr2", "sq_meters"), + self.params1 = OrderedDict([("id_col", "cartodb_id"), + ("attr1", "andy"), + ("attr2", "jay_z"), ("subquery", "SELECT * FROM a_list"), ("geom_col", "the_geom"), ("num_ngbrs", 321)]) + self.params2 = OrderedDict([("id_col", "cartodb_id"), + ("numerator", "price"), + ("denominator", "sq_meters"), + ("subquery", "SELECT * FROM pecan"), + ("geom_col", "the_geom"), + ("num_ngbrs", 321)]) + self.params3 = OrderedDict([("id_col", "cartodb_id"), - ("attr1", "numerator"), - ("attr2", "denominator"), + ("numerator", "sq_meters"), + ("denominator", "price"), ("subquery", "SELECT * FROM pecan"), ("geom_col", "the_geom"), ("num_ngbrs", 321)]) @@ -39,20 +39,20 @@ class PysalUtilsTest(unittest.TestCase): def test_query_attr_select(self): """Test query_attr_select""" - ans = "i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, " + ans1 = ("i.\"andy\"::numeric As attr1, " + "i.\"jay_z\"::numeric As attr2, ") - ans2 = "i.\"price\"::numeric As attr1, " \ - "i.\"sq_meters\"::numeric As attr2, " + ans2 = ("i.\"price\"::numeric As attr1, " + "i.\"sq_meters\"::numeric As attr2, ") - ans3 = "i.\"numerator\"::numeric As attr1, " \ - "i.\"denominator\"::numeric As attr2, " + ans3 = ("i.\"sq_meters\"::numeric As attr1, " + "i.\"price\"::numeric As attr2, ") - ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ - "i.\"_2014_jan\"::numeric As attr2, " \ - "i.\"_2014_feb\"::numeric As attr3, " + ans_array = ("i.\"_2013_dec\"::numeric As attr1, " + "i.\"_2014_jan\"::numeric As attr2, " + "i.\"_2014_feb\"::numeric As attr3, ") - self.assertEqual(pu.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params1), ans1) self.assertEqual(pu.query_attr_select(self.params2), ans2) self.assertEqual(pu.query_attr_select(self.params3), ans3) self.assertEqual(pu.query_attr_select(self.params_array), ans_array) @@ -60,21 +60,20 @@ class PysalUtilsTest(unittest.TestCase): def test_query_attr_where(self): """Test pu.query_attr_where""" - ans = "idx_replace.\"andy\" IS NOT NULL AND " \ - "idx_replace.\"jay_z\" IS NOT NULL AND " \ - "idx_replace.\"jay_z\" <> 0" + ans1 = ("idx_replace.\"andy\" IS NOT NULL AND " + "idx_replace.\"jay_z\" IS NOT NULL") - ans_array = "idx_replace.\"_2013_dec\" IS NOT NULL AND " \ - "idx_replace.\"_2014_jan\" IS NOT NULL AND " \ - "idx_replace.\"_2014_feb\" IS NOT NULL" + ans_array = ("idx_replace.\"_2013_dec\" IS NOT NULL AND " + "idx_replace.\"_2014_jan\" IS NOT NULL AND " + "idx_replace.\"_2014_feb\" IS NOT NULL") - self.assertEqual(pu.query_attr_where(self.params), ans) + self.assertEqual(pu.query_attr_where(self.params1), ans1) self.assertEqual(pu.query_attr_where(self.params_array), ans_array) def test_knn(self): """Test knn neighbors constructor""" - ans = "SELECT i.\"cartodb_id\" As id, " \ + ans1 = "SELECT i.\"cartodb_id\" As id, " \ "i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, " \ "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ @@ -82,15 +81,13 @@ class PysalUtilsTest(unittest.TestCase): "WHERE " \ "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0 " \ + "j.\"jay_z\" IS NOT NULL " \ "ORDER BY " \ "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ "LIMIT 321)) As neighbors " \ "FROM (SELECT * FROM a_list) As i " \ "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 " \ + "i.\"jay_z\" IS NOT NULL " \ "ORDER BY i.\"cartodb_id\" ASC;" ans_array = "SELECT i.\"cartodb_id\" As id, " \ @@ -111,13 +108,13 @@ class PysalUtilsTest(unittest.TestCase): "i.\"_2014_feb\" IS NOT NULL "\ "ORDER BY i.\"cartodb_id\" ASC;" - self.assertEqual(pu.knn(self.params), ans) + self.assertEqual(pu.knn(self.params1), ans1) self.assertEqual(pu.knn(self.params_array), ans_array) def test_queen(self): """Test queen neighbors constructor""" - ans = "SELECT i.\"cartodb_id\" As id, " \ + ans1 = "SELECT i.\"cartodb_id\" As id, " \ "i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, " \ "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ @@ -127,23 +124,21 @@ class PysalUtilsTest(unittest.TestCase): "ST_Touches(i.\"the_geom\", " \ "j.\"the_geom\") AND " \ "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0)" \ + "j.\"jay_z\" IS NOT NULL)" \ ") As neighbors " \ "FROM (SELECT * FROM a_list) As i " \ "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 " \ + "i.\"jay_z\" IS NOT NULL " \ "ORDER BY i.\"cartodb_id\" ASC;" - self.assertEqual(pu.queen(self.params), ans) + self.assertEqual(pu.queen(self.params1), ans1) def test_construct_neighbor_query(self): """Test construct_neighbor_query""" # Compare to raw knn query - self.assertEqual(pu.construct_neighbor_query('knn', self.params), - pu.knn(self.params)) + self.assertEqual(pu.construct_neighbor_query('knn', self.params1), + pu.knn(self.params1)) def test_get_attributes(self): """Test get_attributes""" From f1d420a6f7fc83cfc58862065c3a845bf31a5ba6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 26 Sep 2016 10:11:16 -0400 Subject: [PATCH 31/48] ordering fixes --- .../crankshaft/pysal_utils/pysal_utils.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index f0c73ba..c0ec19d 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -42,15 +42,16 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5): def query_attr_select(params): """ Create portion of SELECT statement for attributes inolved in query. + Defaults to order in the params @param params: dict of information used in query (column names, table name, etc.) Example: - OrderedDict([('attr1', 'numerator'), - ('attr2', 'denominator'), + OrderedDict([('numerator', 'price'), + ('denominator', 'sq_meters'), ('subquery', 'SELECT * FROM interesting_data')]) Output: - "i.\"numerator\"::numeric As attr1, " \ - "i.\"denominator\"::numeric As attr2, " + "i.\"price\"::numeric As attr1, " \ + "i.\"sq_meters\"::numeric As attr2, " """ attr_string = "" @@ -81,7 +82,8 @@ def query_attr_where(params): 'numerator': 'data1', 'denominator': 'data2', '': ...} - Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' + Output: + 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' Input: {'subquery': ..., 'time_cols': ['time1', 'time2', 'time3'], @@ -102,14 +104,16 @@ def query_attr_where(params): ## moran where clauses # get keys - attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]) + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', + 'num_ngbrs', 'subquery')] # add values to template for attr in attrs: attr_string.append(template % params[attr]) - if len(attrs) == 2: - attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]]) + if 'denominator' in attrs: + attr_string.append( + "idx_replace.\"%s\" <> 0" % params['denominator']) out = " AND ".join(attr_string) From 1ef3f864749a8b6f89a0c3a9e35e6ab9ed00f659 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 26 Sep 2016 10:13:27 -0400 Subject: [PATCH 32/48] small updates after ordering fix --- src/pg/test/expected/02_moran_test.out | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index e7aef8b..95ce189 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -197,10 +197,11 @@ code|quads 46|LL 47|LL 48|HH -49|LL +49|LH 50|HH 51|LL -(51 rows) +52|LL +(52 rows) _cdb_random_seeds (1 row) @@ -259,9 +260,10 @@ code|quads 45|LH 46|LL 47|LL -49|LL +49|LH 51|LL -(28 rows) +52|LL +(29 rows) _cdb_random_seeds (1 row) @@ -277,4 +279,5 @@ code|quads 42|LH 43|LH 45|LH -(11 rows) +49|LH +(12 rows) From aaa36569ded69399f73848440e18329f7457354b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 26 Sep 2016 16:26:34 -0400 Subject: [PATCH 33/48] first add --- src/pg/sql/18_outliers.sql | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/pg/sql/18_outliers.sql diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql new file mode 100644 index 0000000..48588ce --- /dev/null +++ b/src/pg/sql/18_outliers.sql @@ -0,0 +1,52 @@ +CREATE OR REPLACE FUNCTION CDB_StaticOutlier(attr numeric, threshold numeric) +RETURNS numeric +AS $$ +BEGIN + + RETURN attr > threshold; + +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION CDB_PercentOutlier(attr numeric[], outlier_fraction numeric, ids int[]) +RETURNS TABLE(outlier boolean, rowid int) +AS $$ +DECLARE + avg_val numeric; + out_vals boolean[]; +BEGIN + + SELECT avg(i) INTO avg_val FROM unnest(attr) As x(i); + + SELECT array_agg( i > avg_val * outlier_fraction) INTO out_vals + FROM unnest(attr) As x(i); + + RETURN QUERY + SELECT unnest(out_vals) As outlier, + unnest(ids) As rowid; + +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(attrs numeric[], num_deviations numeric, ids int[]) +RETURNS TABLE(outlier boolean, rowid int) +AS $$ +DECLARE + stddev_val numeric; + avg_val numeric; + out_vals boolean[]; +BEGIN + + SELECT stddev(i), avg(i) INTO stddev_val, avg_val + FROM unnest(attrs) As x(i); + + SELECT array_agg(abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals + FROM unnest(attrs) As x(i); + + + RETURN QUERY + SELECT unnest(out_vals) As outlier, + unnest(ids) As rowid; +END; +$$ LANGUAGE plpgsql; From f2bb0b496bc9f86d2d09db583cb4677d73d94a50 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 26 Sep 2016 16:51:22 -0400 Subject: [PATCH 34/48] small fixes --- src/pg/sql/18_outliers.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 48588ce..7d4b3b2 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -1,5 +1,8 @@ + +-- Find outliers using a static threshold +-- CREATE OR REPLACE FUNCTION CDB_StaticOutlier(attr numeric, threshold numeric) -RETURNS numeric +RETURNS boolean AS $$ BEGIN @@ -8,6 +11,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Find outliers by a percentage above the threshold CREATE OR REPLACE FUNCTION CDB_PercentOutlier(attr numeric[], outlier_fraction numeric, ids int[]) RETURNS TABLE(outlier boolean, rowid int) @@ -29,6 +33,8 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Find outliers above a given number of standard deviations from the mean + CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(attrs numeric[], num_deviations numeric, ids int[]) RETURNS TABLE(outlier boolean, rowid int) AS $$ From b8accb48fc15fa6ed235700f363e33a7483621c0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 28 Sep 2016 15:55:56 -0400 Subject: [PATCH 35/48] adds tests --- src/pg/sql/18_outliers.sql | 16 +++++-- src/pg/test/expected/18_outliers.out | 16 +++++++ src/pg/test/sql/18_outliers.sql | 70 ++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 src/pg/test/expected/18_outliers.out create mode 100644 src/pg/test/sql/18_outliers.sql diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 7d4b3b2..3ae7a5e 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -12,6 +12,7 @@ END; $$ LANGUAGE plpgsql; -- Find outliers by a percentage above the threshold +-- TODO: add symmetric option? `symmetric boolean DEFAULT false` CREATE OR REPLACE FUNCTION CDB_PercentOutlier(attr numeric[], outlier_fraction numeric, ids int[]) RETURNS TABLE(outlier boolean, rowid int) @@ -21,10 +22,15 @@ DECLARE out_vals boolean[]; BEGIN - SELECT avg(i) INTO avg_val FROM unnest(attr) As x(i); + SELECT avg(i) INTO avg_val + FROM unnest(attr) As x(i); - SELECT array_agg( i > avg_val * outlier_fraction) INTO out_vals - FROM unnest(attr) As x(i); + SELECT array_agg( + CASE WHEN avg_val = 0 THEN null + ELSE outlier_fraction > i / avg_val + END + ) INTO out_vals + FROM unnest(attr) As x(i); RETURN QUERY SELECT unnest(out_vals) As outlier, @@ -45,10 +51,10 @@ DECLARE BEGIN SELECT stddev(i), avg(i) INTO stddev_val, avg_val - FROM unnest(attrs) As x(i); + FROM unnest(attrs) As x(i); SELECT array_agg(abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals - FROM unnest(attrs) As x(i); + FROM unnest(attrs) As x(i); RETURN QUERY diff --git a/src/pg/test/expected/18_outliers.out b/src/pg/test/expected/18_outliers.out new file mode 100644 index 0000000..1c0f186 --- /dev/null +++ b/src/pg/test/expected/18_outliers.out @@ -0,0 +1,16 @@ +outlier|rowid +t|11 +t|16 +t|17 +outlier|rowid +t|16 +t|17 +outlier|rowid +t|8 +t|11 +t|16 +outlier|rowid +t|8 +t|9 +t|11 +t|15 diff --git a/src/pg/test/sql/18_outliers.sql b/src/pg/test/sql/18_outliers.sql new file mode 100644 index 0000000..432d4c1 --- /dev/null +++ b/src/pg/test/sql/18_outliers.sql @@ -0,0 +1,70 @@ +SET client_min_messages TO WARNING; +\set ECHO none +\pset format unaligned + +-- +-- postgres=# select round(avg(i), 3) as avg, +-- round(stddev(i), 3) as stddev, +-- round(avg(i) + stddev(i), 3) as one_stddev, +-- round(avg(i) + 2 * stddev(i), 3) As two_stddev +-- from unnest(ARRAY[1,3,2,3,5,1,2,32,12,3,57,2,1,4,2,100]) As x(i); +-- avg | stddev | one_stddev | two_stddev +-- --------+--------+------------+------------ +-- 14.375 | 27.322 | 41.697 | 69.020 + + +-- With an threshold of 1.0 standard deviation, ids 11, 16, and 17 are outliers +WITH a AS ( + SELECT + ARRAY[1,3,2,3,5,1,2,32,12, 3,57, 2, 1, 4, 2,100,-100]::numeric[] As vals, ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids +), b As ( + SELECT + (cdb_StdDevOutlier(vals, 1.0, ids)).* + FROM a + ORDER BY ids) +SELECT * +FROM b +WHERE outlier IS TRUE; + +-- With a threshold of 2.0 standard deviations, id 16 is the only outlier +WITH a AS ( + SELECT + ARRAY[1,3,2,3,5,1,2,32,12, 3,57, 2, 1, 4, 2,100,-100]::numeric[] As vals, + ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids +), b As ( + SELECT + (CDB_StdDevOutlier(vals, 2.0, ids)).* + FROM a + ORDER BY ids) +SELECT * +FROM b +WHERE outlier IS TRUE; + +-- With a ratio threshold of 2.0 threshold (100% above or below the mean) +-- which is greater than ~21, which are values +WITH a AS ( + SELECT + ARRAY[1,3,2,3,5,1,2,32,12, 3,57, 2, 1, 4, 2,100,-100]::numeric[] As vals, + ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids +), b As ( + SELECT + (CDB_PercentOutlier(vals, 2.0, ids)).* + FROM a + ORDER BY ids) +SELECT * + FROM b + WHERE outlier IS TRUE; + +-- With a static threshold of 11, what are the outliers +WITH a AS ( + SELECT + ARRAY[1,3,2,3,5,1,2,32,12, 3,57, 2, 1, 4, 2,100,-100]::numeric[] As vals, + ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids + ), b As ( + SELECT unnest(vals) As v, unnest(ids) as i + FROM a + ) +SELECT CDB_StaticOutlier(v, 11.0), i + FROM b +WHERE CDB_StaticOutlier(v, 11.0) is True +ORDER BY i; From acde384157bf0ffe516c6ab3c5504e61fd28dce9 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 28 Sep 2016 16:27:41 -0400 Subject: [PATCH 36/48] update tests --- src/pg/sql/18_outliers.sql | 2 +- .../{18_outliers.out => 18_outliers_test.out} | 8 +++++++- .../sql/{18_outliers.sql => 18_outliers_test.sql} | 11 ++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) rename src/pg/test/expected/{18_outliers.out => 18_outliers_test.out} (54%) rename src/pg/test/sql/{18_outliers.sql => 18_outliers_test.sql} (86%) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 3ae7a5e..6b6d943 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -27,7 +27,7 @@ BEGIN SELECT array_agg( CASE WHEN avg_val = 0 THEN null - ELSE outlier_fraction > i / avg_val + ELSE outlier_fraction < i::numeric / avg_val::numeric END ) INTO out_vals FROM unnest(attr) As x(i); diff --git a/src/pg/test/expected/18_outliers.out b/src/pg/test/expected/18_outliers_test.out similarity index 54% rename from src/pg/test/expected/18_outliers.out rename to src/pg/test/expected/18_outliers_test.out index 1c0f186..a329e07 100644 --- a/src/pg/test/expected/18_outliers.out +++ b/src/pg/test/expected/18_outliers_test.out @@ -1,16 +1,22 @@ +SET client_min_messages TO WARNING; +\set ECHO none outlier|rowid t|11 t|16 t|17 +(3 rows) outlier|rowid t|16 t|17 +(2 rows) outlier|rowid t|8 t|11 t|16 +(3 rows) outlier|rowid t|8 t|9 t|11 -t|15 +t|16 +(4 rows) diff --git a/src/pg/test/sql/18_outliers.sql b/src/pg/test/sql/18_outliers_test.sql similarity index 86% rename from src/pg/test/sql/18_outliers.sql rename to src/pg/test/sql/18_outliers_test.sql index 432d4c1..c12c889 100644 --- a/src/pg/test/sql/18_outliers.sql +++ b/src/pg/test/sql/18_outliers_test.sql @@ -19,7 +19,7 @@ WITH a AS ( ARRAY[1,3,2,3,5,1,2,32,12, 3,57, 2, 1, 4, 2,100,-100]::numeric[] As vals, ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids ), b As ( SELECT - (cdb_StdDevOutlier(vals, 1.0, ids)).* + (cdb_crankshaft.cdb_StdDevOutlier(vals, 1.0, ids)).* FROM a ORDER BY ids) SELECT * @@ -33,7 +33,7 @@ WITH a AS ( ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids ), b As ( SELECT - (CDB_StdDevOutlier(vals, 2.0, ids)).* + (cdb_crankshaft.CDB_StdDevOutlier(vals, 2.0, ids)).* FROM a ORDER BY ids) SELECT * @@ -48,7 +48,7 @@ WITH a AS ( ARRAY[1,2,3,4,5,6,7, 8, 9,10,11,12,13,14,15, 16, 17]::int[] As ids ), b As ( SELECT - (CDB_PercentOutlier(vals, 2.0, ids)).* + (cdb_crankshaft.CDB_PercentOutlier(vals, 2.0, ids)).* FROM a ORDER BY ids) SELECT * @@ -64,7 +64,8 @@ WITH a AS ( SELECT unnest(vals) As v, unnest(ids) as i FROM a ) -SELECT CDB_StaticOutlier(v, 11.0), i +SELECT cdb_crankshaft.CDB_StaticOutlier(v, 11.0) As outlier, i As rowid FROM b -WHERE CDB_StaticOutlier(v, 11.0) is True +WHERE cdb_crankshaft.CDB_StaticOutlier(v, 11.0) is True ORDER BY i; + From b54c62890f58281aea399ec2f3d0e271783e8e01 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 08:48:22 -0400 Subject: [PATCH 37/48] adds hand-off doc line --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3de84c4..fbc5c8d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,4 +6,5 @@ - [ ] Video explaining the analysis and showing examples - [ ] Analysis Documentation written [template](https://docs.google.com/a/cartodb.com/document/d/1X2KOtaiEBKWNMp8UjwcLB-kE9aIOw09aOjX3oaCjeME/edit?usp=sharing) - [ ] Smoke test written +- [ ] Hand-off document for camshaft node written From 8bc6f69a1bf992dce2668652e722c42de06715d0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 10:12:32 -0400 Subject: [PATCH 38/48] adding exceptions to improve robustness --- src/pg/sql/18_outliers.sql | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 3ae7a5e..1fa8d93 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -14,7 +14,7 @@ $$ LANGUAGE plpgsql; -- Find outliers by a percentage above the threshold -- TODO: add symmetric option? `symmetric boolean DEFAULT false` -CREATE OR REPLACE FUNCTION CDB_PercentOutlier(attr numeric[], outlier_fraction numeric, ids int[]) +CREATE OR REPLACE FUNCTION CDB_PercentOutlier(column_values numeric[], outlier_fraction numeric, ids int[]) RETURNS TABLE(outlier boolean, rowid int) AS $$ DECLARE @@ -23,14 +23,15 @@ DECLARE BEGIN SELECT avg(i) INTO avg_val - FROM unnest(attr) As x(i); + FROM unnest(column_values) As x(i); + + IF avg_val = 0 THEN + RAISE EXCEPTION 'Mean value is zero. Try another outlier method.'; + END IF; SELECT array_agg( - CASE WHEN avg_val = 0 THEN null - ELSE outlier_fraction > i / avg_val - END - ) INTO out_vals - FROM unnest(attr) As x(i); + outlier_fraction > i / avg_val) INTO out_vals + FROM unnest(column_values) As x(i); RETURN QUERY SELECT unnest(out_vals) As outlier, @@ -53,7 +54,12 @@ BEGIN SELECT stddev(i), avg(i) INTO stddev_val, avg_val FROM unnest(attrs) As x(i); - SELECT array_agg(abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals + IF stddev_val = 0 THEN + RAISE EXCEPTION 'Standard deviation of input data is zero'; + END IF; + + SELECT array_agg( + abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals FROM unnest(attrs) As x(i); From 5754087140da57a5e716a1e259294053ab229ede Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 11:09:10 -0400 Subject: [PATCH 39/48] adds symmetric option for stddev outlier --- src/pg/sql/18_outliers.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 1fa8d93..454e383 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -12,7 +12,7 @@ END; $$ LANGUAGE plpgsql; -- Find outliers by a percentage above the threshold --- TODO: add symmetric option? `symmetric boolean DEFAULT false` +-- TODO: add symmetric option? `is_symmetric boolean DEFAULT false` CREATE OR REPLACE FUNCTION CDB_PercentOutlier(column_values numeric[], outlier_fraction numeric, ids int[]) RETURNS TABLE(outlier boolean, rowid int) @@ -42,7 +42,7 @@ $$ LANGUAGE plpgsql; -- Find outliers above a given number of standard deviations from the mean -CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(attrs numeric[], num_deviations numeric, ids int[]) +CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(attrs numeric[], num_deviations numeric, ids int[], is_symmetric boolean DEFAULT true) RETURNS TABLE(outlier boolean, rowid int) AS $$ DECLARE @@ -58,10 +58,15 @@ BEGIN RAISE EXCEPTION 'Standard deviation of input data is zero'; END IF; - SELECT array_agg( - abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals - FROM unnest(attrs) As x(i); - + IF is_symmetric THEN + SELECT array_agg( + abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals + FROM unnest(attrs) As x(i); + ELSE + SELECT array_agg( + (i - avg_val) / stddev_val > num_deviations) INTO out_vals + FROM unnest(attrs) As x(i); + END IF; RETURN QUERY SELECT unnest(out_vals) As outlier, From bd05e7739d7bafc15ad7d3f0c9045953103f7210 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 11:10:54 -0400 Subject: [PATCH 40/48] add test to produce error --- src/pg/test/sql/18_outliers.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/pg/test/sql/18_outliers.sql b/src/pg/test/sql/18_outliers.sql index 432d4c1..c84c614 100644 --- a/src/pg/test/sql/18_outliers.sql +++ b/src/pg/test/sql/18_outliers.sql @@ -40,6 +40,21 @@ SELECT * FROM b WHERE outlier IS TRUE; +-- With a Stddev of zero, should throw back error +-- With a threshold of 2.0 standard deviations, id 16 is the only outlier +WITH a AS ( + SELECT + ARRAY[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5]::numeric[] As vals, + ARRAY[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]::int[] As ids +), b As ( + SELECT + (CDB_StdDevOutlier(vals, 1.0, ids)).* + FROM a + ORDER BY ids) +SELECT * +FROM b +WHERE outlier IS TRUE; + -- With a ratio threshold of 2.0 threshold (100% above or below the mean) -- which is greater than ~21, which are values WITH a AS ( From 99856ce95608101553ee11402ea6e81822c61e53 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 11:37:22 -0400 Subject: [PATCH 41/48] flip inequality --- src/pg/sql/18_outliers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index 454e383..b726bf6 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -30,7 +30,7 @@ BEGIN END IF; SELECT array_agg( - outlier_fraction > i / avg_val) INTO out_vals + outlier_fraction < i / avg_val) INTO out_vals FROM unnest(column_values) As x(i); RETURN QUERY From 23b2ad57c538b0eba0eaa8dfdf121993464d5c7c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 11:37:42 -0400 Subject: [PATCH 42/48] test updates --- src/pg/test/expected/18_outliers_test.out | 1 + src/pg/test/sql/18_outliers_test.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg/test/expected/18_outliers_test.out b/src/pg/test/expected/18_outliers_test.out index a329e07..417933b 100644 --- a/src/pg/test/expected/18_outliers_test.out +++ b/src/pg/test/expected/18_outliers_test.out @@ -9,6 +9,7 @@ outlier|rowid t|16 t|17 (2 rows) +ERROR: Standard deviation of input data is zero outlier|rowid t|8 t|11 diff --git a/src/pg/test/sql/18_outliers_test.sql b/src/pg/test/sql/18_outliers_test.sql index 6f18bf1..3b56125 100644 --- a/src/pg/test/sql/18_outliers_test.sql +++ b/src/pg/test/sql/18_outliers_test.sql @@ -48,7 +48,7 @@ WITH a AS ( ARRAY[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]::int[] As ids ), b As ( SELECT - (CDB_StdDevOutlier(vals, 1.0, ids)).* + (cdb_crankshaft.CDB_StdDevOutlier(vals, 1.0, ids)).* FROM a ORDER BY ids) SELECT * From 6846014a4fb1ccb1d738cbde48fbc17d33617394 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 29 Sep 2016 11:42:11 -0400 Subject: [PATCH 43/48] adding docs --- doc/18_outliers.md | 163 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 doc/18_outliers.md diff --git a/doc/18_outliers.md b/doc/18_outliers.md new file mode 100644 index 0000000..f1aa862 --- /dev/null +++ b/doc/18_outliers.md @@ -0,0 +1,163 @@ +## Outlier Detection + +This set of functions detects the presence of outliers. There are three functions for finding outliers from non-spatial data: + +1. Static Outliers +1. Percentage Outliers +1. Standard Deviation Outliers + +### CDB_StaticOutlier(column_value numeric, threshold numeric) + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| column_value | numeric | The column of values on which to apply the threshold | +| threshold | numeric | The static threshold which is used to indicate whether a `column_value` is an outlier or not | + +### Returns + +Returns a boolean (true/false) depending on whether a value is above or below (or equal to) the threshold + +| Name | Type | Description | +|------|------|-------------| +| outlier | boolean | classification of whether a row is an outlier or not | + +#### Example Usage + +With a table `website_visits`: + +``` +| id | visits_10k | +|----|------------| +| 1 | 1 | +| 2 | 3 | +| 3 | 5 | +| 4 | 1 | +| 5 | 32 | +| 6 | 3 | +| 7 | 57 | +| 8 | 2 | +``` + +```sql +SELECT + id, + CDB_StaticOutlier(visits_10k, 11.0) As outlier, + visits_10k +FROM website_visits +``` + +``` +| id | outlier | visits_10k | +|----|---------|------------| +| 1 | f | 1 | +| 2 | f | 3 | +| 3 | f | 5 | +| 4 | f | 1 | +| 5 | t | 32 | +| 6 | f | 3 | +| 7 | t | 57 | +| 8 | f | 2 | +``` + +### CDB_PercentOutlier(column_values numeric[], ratio_threshold numeric, ids int[]) + +`CDB_PercentOutlier` calculates whether or not a value falls above a given threshold based on a percentage above the mean value of the input values. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| column_values | numeric[] | An array of the values to calculate the outlier classification on | +| outlier_fraction | numeric | The threshold above which a column value divided by the mean of all values is considered an outlier | +| ids | int[] | An array of the unique row ids of the input data (usually `cartodb_id`) | + +### Returns + +Returns a table of the outlier classification with the following columns + +| Name | Type | Description | +|------|------|-------------| +| outlier | boolean | classification of whether a row is an outlier or not | +| rowid | int | original row id (e.g., input `cartodb_id`) of the row which has the outlier classification | + +#### Example Usage + +This example find outliers which are more than 100% larger than the average (that is, more than 2.0 times larger). + +```sql +WITH cte As ( + SELECT + unnest(Array[1,2,3,4,5,6,7,8]) As id, + unnest(Array[1,3,5,1,32,3,57,2]) As visits_10k + ) +SELECT + (CDB_PercentOutlier(array_agg(visits_10k), 2.0, array_agg(id))).* +FROM cte; +``` + +Output +``` +| outlier | rowid | +|---------+-------| +| f | 1 | +| f | 2 | +| f | 3 | +| f | 4 | +| t | 5 | +| f | 6 | +| t | 7 | +| f | 8 | +``` + +### CDB_StdDevOutlier(column_values numeric[], ratio_threshold numeric, ids int[], is_symmetric boolean DEFAULT true) + +`CDB_StdDevOutlier` calculates whether or not a value falls above or below a given threshold based on the number of standard deviations from the mean. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| column_values | numeric[] | An array of the values to calculate the outlier classification on | +| num_deviations | numeric | The threshold in units of standard deviation | +| ids | int[] | An array of the unique row ids of the input data (usually `cartodb_id`) | +| is_symmetric (optional) | boolean | Consider outliers that are symmetric about the mean (default: true) | + +### Returns + +Returns a table of the outlier classification with the following columns + +| Name | Type | Description | +|------|------|-------------| +| outlier | boolean | classification of whether a row is an outlier or not | +| rowid | int | original row id (e.g., input `cartodb_id`) of the row which has the outlier classification | + +#### Example Usage + +This example find outliers which are more than 100% larger than the average (that is, more than 2.0 times larger). + +```sql +WITH cte As ( + SELECT + unnest(Array[1,2,3,4,5,6,7,8]) As id, + unnest(Array[1,3,5,1,32,3,57,2]) As visits_10k + ) +SELECT + (CDB_StdDevOutlier(array_agg(visits_10k), 2.0, array_agg(id))).* +FROM cte; +``` + +Output +``` +| outlier | rowid | +|---------+-------| +| f | 1 | +| f | 2 | +| f | 3 | +| f | 4 | +| f | 5 | +| f | 6 | +| t | 7 | +| f | 8 | +``` From 0a53a6e71d5e5dceca1ce266b1519d0f9a8df50d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 6 Oct 2016 08:19:57 -0400 Subject: [PATCH 44/48] fix error variable name bug, pep8 updates --- .../crankshaft/space_time_dynamics/markov.py | 47 +++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index ae788d7..b1c09ba 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -8,12 +8,14 @@ import pysal as ps import plpy import crankshaft.pysal_utils as pu + def spatial_markov_trend(subquery, time_cols, num_classes=7, w_type='knn', num_ngbrs=5, permutations=0, geom_col='the_geom', id_col='cartodb_id'): """ Predict the trends of a unit based on: - 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) + 1. history of its transitions to different classes (e.g., 1st quantile + -> 2nd quantile) 2. average class of its neighbors Inputs: @@ -56,16 +58,15 @@ def spatial_markov_trend(subquery, time_cols, num_classes=7, ) if len(query_result) == 0: return zip([None], [None], [None], [None], [None]) - except plpy.SPIError, e: - plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals))) - plpy.error('Analysis failed: %s' % e) + except plpy.SPIError, err: + plpy.error('Analysis failed: %s' % err) return zip([None], [None], [None], [None], [None]) - ## build weight + # build weight weights = pu.get_weight(query_result, w_type) weights.transform = 'r' - ## prep time data + # prep time data t_data = get_time_data(query_result, time_cols) plpy.debug('shape of t_data %d, %d' % t_data.shape) @@ -78,23 +79,26 @@ def spatial_markov_trend(subquery, time_cols, num_classes=7, fixed=False, permutations=permutations) - ## get lag classes + # get lag classes lag_classes = ps.Quantiles( ps.lag_spatial(weights, t_data[:, -1]), k=num_classes).yb - ## look up probablity distribution for each unit according to class and lag class + # look up probablity distribution for each unit according to class and lag + # class prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) - ## find the ups and down and overall distribution of each cell - trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, - sp_markov_result.classes[:, -1]) + # find the ups and down and overall distribution of each cell + trend_up, trend_down, trend, volatility = get_prob_stats( + prob_dist, + sp_markov_result.classes[:, -1]) - ## output the results + # output the results return zip(trend, trend_up, trend_down, volatility, weights.id_order) + def get_time_data(markov_data, time_cols): """ Extract the time columns and bin appropriately @@ -103,7 +107,8 @@ def get_time_data(markov_data, time_cols): return np.array([[x['attr' + str(i)] for x in markov_data] for i in range(1, num_attrs+1)], dtype=float).transpose() -## not currently used + +# not currently used def rebin_data(time_data, num_time_per_bin): """ Convert an n x l matrix into an (n/m) x l matrix where the values are @@ -131,14 +136,16 @@ def rebin_data(time_data, num_time_per_bin): """ if time_data.shape[1] % num_time_per_bin == 0: - ## if fit is perfect, then use it + # if fit is perfect, then use it n_max = time_data.shape[1] / num_time_per_bin else: - ## fit remainders into an additional column + # fit remainders into an additional column n_max = time_data.shape[1] / num_time_per_bin + 1 - return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) - for i in range(n_max)]).T + return np.array( + [time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) + for i in range(n_max)]).T + def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ @@ -157,6 +164,7 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices): return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) + def get_prob_stats(prob_dist, unit_indices): """ get the statistics of the probability distributions @@ -179,11 +187,12 @@ def get_prob_stats(prob_dist, unit_indices): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() trend_down[i] = prob_dist[i, :unit_indices[i]].sum() if prob_dist[i, unit_indices[i]] > 0.0: - trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + trend[i] = ((trend_up[i] - trend_down[i]) / + (prob_dist[i, unit_indices[i]])) else: trend[i] = None - ## calculate volatility of distribution + # calculate volatility of distribution volatility = prob_dist.std(axis=1) return trend_up, trend_down, trend, volatility From 11c33ce3fa99926eeafc6091b321f2fddd827435 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 6 Oct 2016 08:56:14 -0400 Subject: [PATCH 45/48] adds pep8 check item --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fbc5c8d..529ca2c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,4 +7,4 @@ - [ ] Analysis Documentation written [template](https://docs.google.com/a/cartodb.com/document/d/1X2KOtaiEBKWNMp8UjwcLB-kE9aIOw09aOjX3oaCjeME/edit?usp=sharing) - [ ] Smoke test written - [ ] Hand-off document for camshaft node written - +- [ ] If function is in Python, code conforms to [PEP8 Style Guide](https://www.python.org/dev/peps/pep-0008/) From c7f5c2451071e813ede30eff5f5f9f3f005cc7f0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 6 Oct 2016 09:53:22 -0400 Subject: [PATCH 46/48] update signature names --- doc/18_outliers.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/18_outliers.md b/doc/18_outliers.md index f1aa862..f557529 100644 --- a/doc/18_outliers.md +++ b/doc/18_outliers.md @@ -25,7 +25,7 @@ Returns a boolean (true/false) depending on whether a value is above or below (o #### Example Usage -With a table `website_visits`: +With a table `website_visits` and a column of the number of website visits in units of 10,000 visits: ``` | id | visits_10k | @@ -61,7 +61,7 @@ FROM website_visits | 8 | f | 2 | ``` -### CDB_PercentOutlier(column_values numeric[], ratio_threshold numeric, ids int[]) +### CDB_PercentOutlier(column_values numeric[], outlier_fraction numeric, ids int[]) `CDB_PercentOutlier` calculates whether or not a value falls above a given threshold based on a percentage above the mean value of the input values. @@ -79,7 +79,7 @@ Returns a table of the outlier classification with the following columns | Name | Type | Description | |------|------|-------------| -| outlier | boolean | classification of whether a row is an outlier or not | +| is_outlier | boolean | classification of whether a row is an outlier or not | | rowid | int | original row id (e.g., input `cartodb_id`) of the row which has the outlier classification | #### Example Usage @@ -111,7 +111,7 @@ Output | f | 8 | ``` -### CDB_StdDevOutlier(column_values numeric[], ratio_threshold numeric, ids int[], is_symmetric boolean DEFAULT true) +### CDB_StdDevOutlier(column_values numeric[], num_deviations numeric, ids int[], is_symmetric boolean DEFAULT true) `CDB_StdDevOutlier` calculates whether or not a value falls above or below a given threshold based on the number of standard deviations from the mean. @@ -130,7 +130,7 @@ Returns a table of the outlier classification with the following columns | Name | Type | Description | |------|------|-------------| -| outlier | boolean | classification of whether a row is an outlier or not | +| is_outlier | boolean | classification of whether a row is an outlier or not | | rowid | int | original row id (e.g., input `cartodb_id`) of the row which has the outlier classification | #### Example Usage From da1449331cec83b8e3598c63630f63df2510e944 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 6 Oct 2016 09:53:38 -0400 Subject: [PATCH 47/48] update signature variable names --- src/pg/sql/18_outliers.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/pg/sql/18_outliers.sql b/src/pg/sql/18_outliers.sql index b726bf6..5b23c77 100644 --- a/src/pg/sql/18_outliers.sql +++ b/src/pg/sql/18_outliers.sql @@ -1,12 +1,12 @@ -- Find outliers using a static threshold -- -CREATE OR REPLACE FUNCTION CDB_StaticOutlier(attr numeric, threshold numeric) +CREATE OR REPLACE FUNCTION CDB_StaticOutlier(column_value numeric, threshold numeric) RETURNS boolean AS $$ BEGIN - RETURN attr > threshold; + RETURN column_value > threshold; END; $$ LANGUAGE plpgsql; @@ -15,7 +15,7 @@ $$ LANGUAGE plpgsql; -- TODO: add symmetric option? `is_symmetric boolean DEFAULT false` CREATE OR REPLACE FUNCTION CDB_PercentOutlier(column_values numeric[], outlier_fraction numeric, ids int[]) -RETURNS TABLE(outlier boolean, rowid int) +RETURNS TABLE(is_outlier boolean, rowid int) AS $$ DECLARE avg_val numeric; @@ -34,7 +34,7 @@ BEGIN FROM unnest(column_values) As x(i); RETURN QUERY - SELECT unnest(out_vals) As outlier, + SELECT unnest(out_vals) As is_outlier, unnest(ids) As rowid; END; @@ -42,8 +42,8 @@ $$ LANGUAGE plpgsql; -- Find outliers above a given number of standard deviations from the mean -CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(attrs numeric[], num_deviations numeric, ids int[], is_symmetric boolean DEFAULT true) -RETURNS TABLE(outlier boolean, rowid int) +CREATE OR REPLACE FUNCTION CDB_StdDevOutlier(column_values numeric[], num_deviations numeric, ids int[], is_symmetric boolean DEFAULT true) +RETURNS TABLE(is_outlier boolean, rowid int) AS $$ DECLARE stddev_val numeric; @@ -52,7 +52,7 @@ DECLARE BEGIN SELECT stddev(i), avg(i) INTO stddev_val, avg_val - FROM unnest(attrs) As x(i); + FROM unnest(column_values) As x(i); IF stddev_val = 0 THEN RAISE EXCEPTION 'Standard deviation of input data is zero'; @@ -61,15 +61,15 @@ BEGIN IF is_symmetric THEN SELECT array_agg( abs(i - avg_val) / stddev_val > num_deviations) INTO out_vals - FROM unnest(attrs) As x(i); + FROM unnest(column_values) As x(i); ELSE SELECT array_agg( (i - avg_val) / stddev_val > num_deviations) INTO out_vals - FROM unnest(attrs) As x(i); + FROM unnest(column_values) As x(i); END IF; RETURN QUERY - SELECT unnest(out_vals) As outlier, + SELECT unnest(out_vals) As is_outlier, unnest(ids) As rowid; END; $$ LANGUAGE plpgsql; From c7e690980f6165c303de91edc7a463c1497971b6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 6 Oct 2016 10:29:52 -0400 Subject: [PATCH 48/48] update column names in tests --- src/pg/test/expected/18_outliers_test.out | 8 ++++---- src/pg/test/sql/18_outliers_test.sql | 11 +++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/pg/test/expected/18_outliers_test.out b/src/pg/test/expected/18_outliers_test.out index 417933b..0798e0c 100644 --- a/src/pg/test/expected/18_outliers_test.out +++ b/src/pg/test/expected/18_outliers_test.out @@ -1,21 +1,21 @@ SET client_min_messages TO WARNING; \set ECHO none -outlier|rowid +is_outlier|rowid t|11 t|16 t|17 (3 rows) -outlier|rowid +is_outlier|rowid t|16 t|17 (2 rows) ERROR: Standard deviation of input data is zero -outlier|rowid +is_outlier|rowid t|8 t|11 t|16 (3 rows) -outlier|rowid +is_outlier|rowid t|8 t|9 t|11 diff --git a/src/pg/test/sql/18_outliers_test.sql b/src/pg/test/sql/18_outliers_test.sql index 3b56125..9a6b87d 100644 --- a/src/pg/test/sql/18_outliers_test.sql +++ b/src/pg/test/sql/18_outliers_test.sql @@ -24,7 +24,7 @@ WITH a AS ( ORDER BY ids) SELECT * FROM b -WHERE outlier IS TRUE; +WHERE is_outlier IS TRUE; -- With a threshold of 2.0 standard deviations, id 16 is the only outlier WITH a AS ( @@ -38,7 +38,7 @@ WITH a AS ( ORDER BY ids) SELECT * FROM b -WHERE outlier IS TRUE; +WHERE is_outlier IS TRUE; -- With a Stddev of zero, should throw back error -- With a threshold of 2.0 standard deviations, id 16 is the only outlier @@ -53,7 +53,7 @@ WITH a AS ( ORDER BY ids) SELECT * FROM b -WHERE outlier IS TRUE; +WHERE is_outlier IS TRUE; -- With a ratio threshold of 2.0 threshold (100% above or below the mean) -- which is greater than ~21, which are values @@ -68,7 +68,7 @@ WITH a AS ( ORDER BY ids) SELECT * FROM b - WHERE outlier IS TRUE; + WHERE is_outlier IS TRUE; -- With a static threshold of 11, what are the outliers WITH a AS ( @@ -79,8 +79,7 @@ WITH a AS ( SELECT unnest(vals) As v, unnest(ids) as i FROM a ) -SELECT cdb_crankshaft.CDB_StaticOutlier(v, 11.0) As outlier, i As rowid +SELECT cdb_crankshaft.CDB_StaticOutlier(v, 11.0) As is_outlier, i As rowid FROM b WHERE cdb_crankshaft.CDB_StaticOutlier(v, 11.0) is True ORDER BY i; -