cartodb4/crankshaft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moves getis to class-based framework

Browse Source
This commit is contained in:
Andy Eschbacher
2016-12-01 17:06:21 -05:00
parent 2c6fcfc294
commit 59dc9434f7
7 changed files with 68 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/pg/sql/16_getis.sql
View File

@@ -11,8 +11,9 @@ CREATE OR REPLACE FUNCTION
id_col TEXT DEFAULT 'cartodb_id')
RETURNS TABLE (z_score NUMERIC, p_value NUMERIC, p_z_sim NUMERIC, rowid BIGINT)
AS $$
from crankshaft.clustering import getis_ord
return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
from crankshaft.clustering import Getis
getis = Getis()
return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu;
-- TODO: make a version that accepts the values as arrays

16
src/py/crankshaft/crankshaft/analysis_data_provider.py
View File

@@ -4,7 +4,21 @@ import pysal_utils as pu
class AnalysisDataProvider:
def get_getis(self, w_type, params):
"""fetch data for getis ord's g"""
try:
query = pu.construct_neighbor_query(w_type, params)
result = plpy.execute(query)
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(4)
else:
return result
except plpy.SPIError, err:
plpy.error('Analysis failed: %s' % err)
def get_markov(self, w_type, params):
"""fetch data for spatial markov"""
try:
query = pu.construct_neighbor_query(w_type, params)
data = plpy.execute(query)
@@ -50,4 +64,4 @@ class AnalysisDataProvider:
data = plpy.execute(query)
return data
except plpy.SPIError, err:
plpy.error("Analysis failed: %s" % err)
plpy.error('Analysis failed: %s' % err)

64
src/py/crankshaft/crankshaft/clustering/getis.py
View File

@@ -3,50 +3,48 @@ Getis-Ord's G geostatistics (hotspot/coldspot analysis)
"""
import pysal as ps
import plpy
from collections import OrderedDict
# crankshaft module
# crankshaft modules
import crankshaft.pysal_utils as pu
from crankshaft.analysis_data_provider import AnalysisDataProvider
# High level interface ---------------------------------------
def getis_ord(subquery, attr,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
Getis-Ord's G*
Implementation building neighbors with a PostGIS database and PySAL's
Getis-Ord's G* hotspot/coldspot module.
Andy Eschbacher
"""
class Getis:
def __init__(self, data_provider=None):
if data_provider is None:
self.data_provider = AnalysisDataProvider()
else:
self.data_provider = data_provider
# geometries with attributes that are null are ignored
# resulting in a collection of not as near neighbors if kNN is chosen
def getis_ord(self, subquery, attr,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
Getis-Ord's G*
Implementation building neighbors with a PostGIS database and PySAL's
Getis-Ord's G* hotspot/coldspot module.
Andy Eschbacher
"""
qvals = OrderedDict([("id_col", id_col),
("attr1", attr),
("geom_col", geom_col),
("subquery", subquery),
("num_ngbrs", num_ngbrs)])
# geometries with attributes that are null are ignored
# resulting in a collection of not as near neighbors if kNN is chosen
query = pu.construct_neighbor_query(w_type, qvals)
qvals = OrderedDict([("id_col", id_col),
("attr1", attr),
("geom_col", geom_col),
("subquery", subquery),
("num_ngbrs", num_ngbrs)])
try:
result = plpy.execute(query)
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(4)
except plpy.SPIError, err:
plpy.error('Query failed: %s' % err)
result = self.data_provider.get_getis(w_type, qvals)
attr_vals = pu.get_attributes(result)
attr_vals = pu.get_attributes(result)
# build PySAL weight object
weight = pu.get_weight(result, w_type, num_ngbrs)
# build PySAL weight object
weight = pu.get_weight(result, w_type, num_ngbrs)
# calculate Getis-Ord's G* z- and p-values
getis = ps.esda.getisord.G_Local(attr_vals, weight,
star=True, permutations=permutations)
# calculate Getis-Ord's G* z- and p-values
getis = ps.esda.getisord.G_Local(attr_vals, weight,
star=True, permutations=permutations)
return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order)
return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order)

1
src/py/crankshaft/crankshaft/clustering/moran.py
View File

@@ -6,7 +6,6 @@ Moran's I geostatistics (global clustering & outliers presence)
# average of the their neighborhood
import pysal as ps
import plpy
from collections import OrderedDict
from crankshaft.analysis_data_provider import AnalysisDataProvider

1
src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
View File

@@ -2,6 +2,7 @@
Spatial dynamics measurements using Spatial Markov
"""
# TODO: remove all plpy dependencies
import numpy as np
import pysal as ps

30
src/py/crankshaft/test/test_clustering_getis.py
View File

@@ -1,18 +1,13 @@
import unittest
import numpy as np
from helper import fixture_file
# from mock_plpy import MockPlPy
# plpy = MockPlPy()
#
# import sys
# sys.modules['plpy'] = plpy
from helper import plpy, fixture_file
import crankshaft.clustering as cc
from crankshaft.clustering import Getis
import crankshaft.pysal_utils as pu
from crankshaft import random_seeds
import json
from crankshaft.analysis_data_provider import AnalysisDataProvider
# Fixture files produced as follows
#
@@ -42,6 +37,14 @@ import json
# lgstar_queen.p_sim, lgstar_queen.p_z_sim)))
class FakeDataProvider(AnalysisDataProvider):
def __init__(self, mock_data):
self.mock_result = mock_data
def get_getis(self, w_type, param):
return self.mock_result
class GetisTest(unittest.TestCase):
"""Testing class for Getis-Ord's G* funtion
This test replicates the work done in PySAL documentation:
@@ -49,8 +52,6 @@ class GetisTest(unittest.TestCase):
"""
def setUp(self):
plpy._reset()
# load raw data for analysis
self.neighbors_data = json.loads(
open(fixture_file('neighbors_getis.json')).read())
@@ -64,10 +65,13 @@ class GetisTest(unittest.TestCase):
data = [{'id': d['id'],
'attr1': d['value'],
'neighbors': d['neighbors']} for d in self.neighbors_data]
plpy._define_result('select', data)
random_seeds.set_random_seeds(1234)
result = cc.getis_ord('subquery', 'value',
'queen', None, 999, 'the_geom', 'cartodb_id')
getis = Getis(FakeDataProvider(data))
result = getis.getis_ord('subquery', 'value',
'queen', None, 999, 'the_geom',
'cartodb_id')
result = [(row[0], row[1]) for row in result]
expected = np.array(self.getis_data)[:, 0:2]
for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected):

2
src/py/crankshaft/test/test_clustering_moran.py
View File

@@ -3,7 +3,7 @@ import numpy as np
from helper import fixture_file
from crankshaft.clustering import Moran
from crankshaft.clustering import AnalysisDataProvider
from crankshaft.analysis_data_provider import AnalysisDataProvider
import crankshaft.pysal_utils as pu
from crankshaft import random_seeds
import json