From 59dc9434f710f432c391de79281e2bc4c1d9c2e4 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 1 Dec 2016 17:06:21 -0500 Subject: [PATCH] moves getis to class-based framework --- src/pg/sql/16_getis.sql | 5 +- .../crankshaft/analysis_data_provider.py | 16 ++++- .../crankshaft/crankshaft/clustering/getis.py | 64 +++++++++---------- .../crankshaft/crankshaft/clustering/moran.py | 1 - .../crankshaft/space_time_dynamics/markov.py | 1 + .../crankshaft/test/test_clustering_getis.py | 30 +++++---- .../crankshaft/test/test_clustering_moran.py | 2 +- 7 files changed, 68 insertions(+), 51 deletions(-) diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index 578f15a..e520227 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -11,8 +11,9 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (z_score NUMERIC, p_value NUMERIC, p_z_sim NUMERIC, rowid BIGINT) AS $$ - from crankshaft.clustering import getis_ord - return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + from crankshaft.clustering import Getis + getis = Getis() + return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- TODO: make a version that accepts the values as arrays diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 1d1cf2b..cbc27bc 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -4,7 +4,21 @@ import pysal_utils as pu class AnalysisDataProvider: + def get_getis(self, w_type, params): + """fetch data for getis ord's g""" + try: + query = pu.construct_neighbor_query(w_type, params) + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + else: + return result + except plpy.SPIError, err: + plpy.error('Analysis failed: %s' % err) + def get_markov(self, w_type, params): + """fetch data for spatial markov""" try: query = pu.construct_neighbor_query(w_type, params) data = plpy.execute(query) @@ -50,4 +64,4 @@ class AnalysisDataProvider: data = plpy.execute(query) return data except plpy.SPIError, err: - plpy.error("Analysis failed: %s" % err) + plpy.error('Analysis failed: %s' % err) diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index a593e64..bef8f50 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -3,50 +3,48 @@ Getis-Ord's G geostatistics (hotspot/coldspot analysis) """ import pysal as ps -import plpy from collections import OrderedDict -# crankshaft module +# crankshaft modules import crankshaft.pysal_utils as pu +from crankshaft.analysis_data_provider import AnalysisDataProvider # High level interface --------------------------------------- -def getis_ord(subquery, attr, - w_type, num_ngbrs, permutations, geom_col, id_col): - """ - Getis-Ord's G* - Implementation building neighbors with a PostGIS database and PySAL's - Getis-Ord's G* hotspot/coldspot module. - Andy Eschbacher - """ +class Getis: + def __init__(self, data_provider=None): + if data_provider is None: + self.data_provider = AnalysisDataProvider() + else: + self.data_provider = data_provider - # geometries with attributes that are null are ignored - # resulting in a collection of not as near neighbors if kNN is chosen + def getis_ord(self, subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Getis-Ord's G* + Implementation building neighbors with a PostGIS database and PySAL's + Getis-Ord's G* hotspot/coldspot module. + Andy Eschbacher + """ - qvals = OrderedDict([("id_col", id_col), - ("attr1", attr), - ("geom_col", geom_col), - ("subquery", subquery), - ("num_ngbrs", num_ngbrs)]) + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors if kNN is chosen - query = pu.construct_neighbor_query(w_type, qvals) + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) - try: - result = plpy.execute(query) - # if there are no neighbors, exit - if len(result) == 0: - return pu.empty_zipped_array(4) - except plpy.SPIError, err: - plpy.error('Query failed: %s' % err) + result = self.data_provider.get_getis(w_type, qvals) + attr_vals = pu.get_attributes(result) - attr_vals = pu.get_attributes(result) + # build PySAL weight object + weight = pu.get_weight(result, w_type, num_ngbrs) - # build PySAL weight object - weight = pu.get_weight(result, w_type, num_ngbrs) + # calculate Getis-Ord's G* z- and p-values + getis = ps.esda.getisord.G_Local(attr_vals, weight, + star=True, permutations=permutations) - # calculate Getis-Ord's G* z- and p-values - getis = ps.esda.getisord.G_Local(attr_vals, weight, - star=True, permutations=permutations) - - return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order) + return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 70a8501..a42a981 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -6,7 +6,6 @@ Moran's I geostatistics (global clustering & outliers presence) # average of the their neighborhood import pysal as ps -import plpy from collections import OrderedDict from crankshaft.analysis_data_provider import AnalysisDataProvider diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index f0c0b4a..3ad8273 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -2,6 +2,7 @@ Spatial dynamics measurements using Spatial Markov """ +# TODO: remove all plpy dependencies import numpy as np import pysal as ps diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py index 835a121..61add11 100644 --- a/src/py/crankshaft/test/test_clustering_getis.py +++ b/src/py/crankshaft/test/test_clustering_getis.py @@ -1,18 +1,13 @@ import unittest import numpy as np +from helper import fixture_file -# from mock_plpy import MockPlPy -# plpy = MockPlPy() -# -# import sys -# sys.modules['plpy'] = plpy -from helper import plpy, fixture_file - -import crankshaft.clustering as cc +from crankshaft.clustering import Getis import crankshaft.pysal_utils as pu from crankshaft import random_seeds import json +from crankshaft.analysis_data_provider import AnalysisDataProvider # Fixture files produced as follows # @@ -42,6 +37,14 @@ import json # lgstar_queen.p_sim, lgstar_queen.p_z_sim))) +class FakeDataProvider(AnalysisDataProvider): + def __init__(self, mock_data): + self.mock_result = mock_data + + def get_getis(self, w_type, param): + return self.mock_result + + class GetisTest(unittest.TestCase): """Testing class for Getis-Ord's G* funtion This test replicates the work done in PySAL documentation: @@ -49,8 +52,6 @@ class GetisTest(unittest.TestCase): """ def setUp(self): - plpy._reset() - # load raw data for analysis self.neighbors_data = json.loads( open(fixture_file('neighbors_getis.json')).read()) @@ -64,10 +65,13 @@ class GetisTest(unittest.TestCase): data = [{'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors']} for d in self.neighbors_data] - plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) - result = cc.getis_ord('subquery', 'value', - 'queen', None, 999, 'the_geom', 'cartodb_id') + getis = Getis(FakeDataProvider(data)) + + result = getis.getis_ord('subquery', 'value', + 'queen', None, 999, 'the_geom', + 'cartodb_id') result = [(row[0], row[1]) for row in result] expected = np.array(self.getis_data)[:, 0:2] for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected): diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 5c8c5c9..cc1930e 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -3,7 +3,7 @@ import numpy as np from helper import fixture_file from crankshaft.clustering import Moran -from crankshaft.clustering import AnalysisDataProvider +from crankshaft.analysis_data_provider import AnalysisDataProvider import crankshaft.pysal_utils as pu from crankshaft import random_seeds import json