From d679975f72216476e5882cd22fc22d2021aa7a7e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 13:53:37 -0500 Subject: [PATCH 01/14] catch empty return values and error on them --- .../crankshaft/analysis_data_provider.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index cbc27bc..373c100 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -2,18 +2,25 @@ import plpy import pysal_utils as pu +NULL_VALUE_ERROR = ('No usable data passed to analysis. Check your input rows ' + 'for null values and fill in appropriately.') + + +def verify_data(n_rows): + if n_rows == 0: + plpy.error(NULL_VALUE_ERROR) + class AnalysisDataProvider: def get_getis(self, w_type, params): """fetch data for getis ord's g""" try: query = pu.construct_neighbor_query(w_type, params) - result = plpy.execute(query) - # if there are no neighbors, exit - if len(result) == 0: - return pu.empty_zipped_array(4) - else: - return result + data = plpy.execute(query) + + # if there are no neighbors or all nulls, exit + verify_data(len(data)) + return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) @@ -23,9 +30,7 @@ class AnalysisDataProvider: query = pu.construct_neighbor_query(w_type, params) data = plpy.execute(query) - if len(data) == 0: - return pu.empty_zipped_array(4) - + verify_data(len(data)) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) @@ -37,8 +42,7 @@ class AnalysisDataProvider: data = plpy.execute(query) # if there are no neighbors, exit - if len(data) == 0: - return pu.empty_zipped_array(2) + verify_data(len(data)) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % e) @@ -48,6 +52,7 @@ class AnalysisDataProvider: """fetch data for non-spatial kmeans""" try: data = plpy.execute(query) + verify_data(len(data)) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) @@ -55,13 +60,14 @@ class AnalysisDataProvider: def get_spatial_kmeans(self, params): """fetch data for spatial kmeans""" query = ("SELECT " - "array_agg({id_col} ORDER BY {id_col}) as ids," - "array_agg(ST_X({geom_col}) ORDER BY {id_col}) As xs," - "array_agg(ST_Y({geom_col}) ORDER BY {id_col}) As ys " + "array_agg(\"{id_col}\" ORDER BY \"{id_col}\") as ids," + "array_agg(ST_X(\"{geom_col}\") ORDER BY \"{id_col}\") As xs," + "array_agg(ST_Y(\"{geom_col}\") ORDER BY \"{id_col}\") As ys " "FROM ({subquery}) As a " - "WHERE {geom_col} IS NOT NULL").format(**params) + "WHERE \"{geom_col}\" IS NOT NULL").format(**params) try: data = plpy.execute(query) + verify_data(len(data)) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) From 10ce109d096c9c2c027700a1c4a9ccf744af3328 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 14:37:07 -0500 Subject: [PATCH 02/14] fix typo on error return --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 373c100..ad8d9c0 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -45,7 +45,7 @@ class AnalysisDataProvider: verify_data(len(data)) return data except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % e) + plpy.error('Analysis failed: %s' % err) return pu.empty_zipped_array(2) def get_nonspatial_kmeans(self, query): From c114ccea339efa21fb608cc60c8e55b62bc87047 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 14:37:31 -0500 Subject: [PATCH 03/14] add condition on null-valued geometries, ref: #143 --- .../crankshaft/crankshaft/pysal_utils/pysal_utils.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 0be95c7..35cfec1 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -25,13 +25,6 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5): Construct PySAL weight from return value of query @param query_res dict-like: query results with attributes and neighbors """ - # if w_type.lower() == 'knn': - # row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs - # weights = {x['id']: row_normed_weights for x in query_res} - # else: - # weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - # if len(x['neighbors']) > 0 - # else [] for x in query_res} neighbors = {x['id']: x['neighbors'] for x in query_res} print 'len of neighbors: %d' % len(neighbors) @@ -146,14 +139,15 @@ def knn(params): "FROM ({subquery}) As j " \ "WHERE " \ "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ - "%(attr_where_j)s " \ + "%(attr_where_j)s AND " \ + "j.\"{geom_col}\" IS NOT NULL " \ "ORDER BY " \ "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ "LIMIT {num_ngbrs})" \ ") As neighbors " \ "FROM ({subquery}) As i " \ "WHERE " \ - "%(attr_where_i)s " \ + "%(attr_where_i)s AND i.\"{geom_col}\" IS NOT NULL " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements return query.format(**params) From ca7a2d6e363db7500ba56d5985be5536364c8520 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 15:00:59 -0500 Subject: [PATCH 04/14] update verify_data to get full data reference --- .../crankshaft/analysis_data_provider.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index ad8d9c0..5f52ff7 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -6,8 +6,8 @@ NULL_VALUE_ERROR = ('No usable data passed to analysis. Check your input rows ' 'for null values and fill in appropriately.') -def verify_data(n_rows): - if n_rows == 0: +def verify_data(data): + if len(data) == 0: plpy.error(NULL_VALUE_ERROR) @@ -19,7 +19,7 @@ class AnalysisDataProvider: data = plpy.execute(query) # if there are no neighbors or all nulls, exit - verify_data(len(data)) + verify_data(data) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) @@ -30,7 +30,7 @@ class AnalysisDataProvider: query = pu.construct_neighbor_query(w_type, params) data = plpy.execute(query) - verify_data(len(data)) + verify_data(data) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) @@ -42,32 +42,33 @@ class AnalysisDataProvider: data = plpy.execute(query) # if there are no neighbors, exit - verify_data(len(data)) + verify_data(data) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) - return pu.empty_zipped_array(2) def get_nonspatial_kmeans(self, query): """fetch data for non-spatial kmeans""" try: data = plpy.execute(query) - verify_data(len(data)) + verify_data(data) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) def get_spatial_kmeans(self, params): """fetch data for spatial kmeans""" - query = ("SELECT " - "array_agg(\"{id_col}\" ORDER BY \"{id_col}\") as ids," - "array_agg(ST_X(\"{geom_col}\") ORDER BY \"{id_col}\") As xs," - "array_agg(ST_Y(\"{geom_col}\") ORDER BY \"{id_col}\") As ys " - "FROM ({subquery}) As a " - "WHERE \"{geom_col}\" IS NOT NULL").format(**params) + query = ''' + SELECT + array_agg("{id_col}" ORDER BY "{id_col}") as ids, + array_agg(ST_X("{geom_col}") ORDER BY "{id_col}") As xs, + array_agg(ST_Y("{geom_col}") ORDER BY "{id_col}") As ys + FROM ({subquery}) As a + WHERE "{geom_col}" IS NOT NULL + '''.format(**params) try: data = plpy.execute(query) - verify_data(len(data)) + verify_data(data) return data except plpy.SPIError, err: plpy.error('Analysis failed: %s' % err) From 50f6ef0fcc6948c66b37c84b4333b0c476595290 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 15:01:44 -0500 Subject: [PATCH 05/14] remove unnecessary code / tests --- .../crankshaft/pysal_utils/pysal_utils.py | 20 ++--- src/py/crankshaft/test/test_pysal_utils.py | 79 +------------------ 2 files changed, 11 insertions(+), 88 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 35cfec1..3cbb7f7 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -193,13 +193,13 @@ def get_attributes(query_res, attr_num=1): dtype=np.float) -def empty_zipped_array(num_nones): - """ - prepare return values for cases of empty weights objects (no neighbors) - Input: - @param num_nones int: number of columns (e.g., 4) - Output: - [(None, None, None, None)] - """ - - return [tuple([None] * num_nones)] +# def empty_zipped_array(num_nones): +# """ +# prepare return values for cases of empty weights objects (no neighbors) +# Input: +# @param num_nones int: number of columns (e.g., 4) +# Output: +# [(None, None, None, None)] +# """ +# +# return [tuple([None] * num_nones)] diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 92b528b..be45164 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -70,80 +70,10 @@ class PysalUtilsTest(unittest.TestCase): self.assertEqual(pu.query_attr_where(self.params1), ans1) self.assertEqual(pu.query_attr_where(self.params_array), ans_array) - def test_knn(self): - """Test knn neighbors constructor""" - - ans1 = "SELECT i.\"cartodb_id\" As id, " \ - "i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, " \ - "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j " \ - "WHERE " \ - "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ - "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL " \ - "ORDER BY " \ - "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ - "LIMIT 321)) As neighbors " \ - "FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL " \ - "ORDER BY i.\"cartodb_id\" ASC;" - - ans_array = "SELECT i.\"cartodb_id\" As id, " \ - "i.\"_2013_dec\"::numeric As attr1, " \ - "i.\"_2014_jan\"::numeric As attr2, " \ - "i.\"_2014_feb\"::numeric As attr3, " \ - "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j " \ - "WHERE i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ - "j.\"_2013_dec\" IS NOT NULL AND " \ - "j.\"_2014_jan\" IS NOT NULL AND " \ - "j.\"_2014_feb\" IS NOT NULL " \ - "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC " \ - "LIMIT 321)) As neighbors " \ - "FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"_2013_dec\" IS NOT NULL AND " \ - "i.\"_2014_jan\" IS NOT NULL AND " \ - "i.\"_2014_feb\" IS NOT NULL "\ - "ORDER BY i.\"cartodb_id\" ASC;" - - self.assertEqual(pu.knn(self.params1), ans1) - self.assertEqual(pu.knn(self.params_array), ans_array) - - def test_queen(self): - """Test queen neighbors constructor""" - - ans1 = "SELECT i.\"cartodb_id\" As id, " \ - "i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, " \ - "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j " \ - "WHERE " \ - "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ - "ST_Touches(i.\"the_geom\", " \ - "j.\"the_geom\") AND " \ - "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL)" \ - ") As neighbors " \ - "FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL " \ - "ORDER BY i.\"cartodb_id\" ASC;" - - self.assertEqual(pu.queen(self.params1), ans1) - - def test_construct_neighbor_query(self): - """Test construct_neighbor_query""" - - # Compare to raw knn query - self.assertEqual(pu.construct_neighbor_query('knn', self.params1), - pu.knn(self.params1)) - def test_get_attributes(self): """Test get_attributes""" - ## need to add tests + # need to add tests self.assertEqual(True, True) @@ -151,10 +81,3 @@ class PysalUtilsTest(unittest.TestCase): """Test get_weight""" self.assertEqual(True, True) - - def test_empty_zipped_array(self): - """Test empty_zipped_array""" - ans2 = [(None, None)] - ans4 = [(None, None, None, None)] - self.assertEqual(pu.empty_zipped_array(2), ans2) - self.assertEqual(pu.empty_zipped_array(4), ans4) From e456158cbfae6f1c807a3f58b2002aa61ca0246f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 10 Jan 2017 15:12:24 -0500 Subject: [PATCH 06/14] removes unneeded function / multilines some queries --- .../crankshaft/pysal_utils/pysal_utils.py | 71 ++++++++----------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 3cbb7f7..4906d1a 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -132,23 +132,21 @@ def knn(params): "attr_where_i": attr_where.replace("idx_replace", "i"), "attr_where_j": attr_where.replace("idx_replace", "j")} - query = "SELECT " \ - "i.\"{id_col}\" As id, " \ - "%(attr_select)s" \ - "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM ({subquery}) As j " \ - "WHERE " \ - "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ - "%(attr_where_j)s AND " \ - "j.\"{geom_col}\" IS NOT NULL " \ - "ORDER BY " \ - "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ - "LIMIT {num_ngbrs})" \ - ") As neighbors " \ - "FROM ({subquery}) As i " \ - "WHERE " \ - "%(attr_where_i)s AND i.\"{geom_col}\" IS NOT NULL " \ - "ORDER BY i.\"{id_col}\" ASC;" % replacements + query = ''' + SELECT + i."{id_col}" As id, + %(attr_select)s + (SELECT ARRAY(SELECT j."{id_col}" + FROM ({subquery}) As j + WHERE i."{id_col}" <> j."{id_col}" AND + %(attr_where_j)s AND + j."{geom_col}" IS NOT NULL + ORDER BY j."{geom_col}" <-> i."{geom_col}" ASC + LIMIT {num_ngbrs})) As neighbors + FROM ({subquery}) As i + WHERE %(attr_where_i)s AND i."{geom_col}" IS NOT NULL + ORDER BY i."{id_col}" ASC; + ''' % replacements return query.format(**params) @@ -165,19 +163,20 @@ def queen(params): "attr_where_i": attr_where.replace("idx_replace", "i"), "attr_where_j": attr_where.replace("idx_replace", "j")} - query = "SELECT " \ - "i.\"{id_col}\" As id, " \ - "%(attr_select)s" \ - "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM ({subquery}) As j " \ - "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ - "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ - "%(attr_where_j)s)" \ - ") As neighbors " \ - "FROM ({subquery}) As i " \ - "WHERE " \ - "%(attr_where_i)s " \ - "ORDER BY i.\"{id_col}\" ASC;" % replacements + query = ''' + SELECT + i."{id_col}" As id, + %(attr_select)s + (SELECT ARRAY(SELECT j."{id_col}" + FROM ({subquery}) As j + WHERE i."{id_col}" <> j."{id_col}" AND + ST_Touches(i."{geom_col}", j."{geom_col}") AND + %(attr_where_j)s)) As neighbors + FROM ({subquery}) As i + WHERE + %(attr_where_i)s + ORDER BY i."{id_col}" ASC; + ''' % replacements return query.format(**params) @@ -191,15 +190,3 @@ def get_attributes(query_res, attr_num=1): """ return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) - - -# def empty_zipped_array(num_nones): -# """ -# prepare return values for cases of empty weights objects (no neighbors) -# Input: -# @param num_nones int: number of columns (e.g., 4) -# Output: -# [(None, None, None, None)] -# """ -# -# return [tuple([None] * num_nones)] From 7322931ca1f52dafb65f1ca4c9fe5a10d74227cb Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 12 Jan 2017 12:00:36 -0500 Subject: [PATCH 07/14] classes to inherit from objects --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 2 +- src/py/crankshaft/crankshaft/clustering/getis.py | 2 +- src/py/crankshaft/crankshaft/clustering/kmeans.py | 2 +- src/py/crankshaft/crankshaft/clustering/moran.py | 2 +- src/py/crankshaft/crankshaft/random_seeds.py | 1 + src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 5f52ff7..25e30fc 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -11,7 +11,7 @@ def verify_data(data): plpy.error(NULL_VALUE_ERROR) -class AnalysisDataProvider: +class AnalysisDataProvider(object): def get_getis(self, w_type, params): """fetch data for getis ord's g""" try: diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index bef8f50..f560e9c 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -12,7 +12,7 @@ from crankshaft.analysis_data_provider import AnalysisDataProvider # High level interface --------------------------------------- -class Getis: +class Getis(object): def __init__(self, data_provider=None): if data_provider is None: self.data_provider = AnalysisDataProvider() diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py index 1e49115..6c1115a 100644 --- a/src/py/crankshaft/crankshaft/clustering/kmeans.py +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -4,7 +4,7 @@ import numpy as np from crankshaft.analysis_data_provider import AnalysisDataProvider -class Kmeans: +class Kmeans(object): def __init__(self, data_provider=None): if data_provider is None: self.data_provider = AnalysisDataProvider() diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index a42a981..b948e04 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -15,7 +15,7 @@ import crankshaft.pysal_utils as pu # High level interface --------------------------------------- -class Moran: +class Moran(object): def __init__(self, data_provider=None): if data_provider is None: self.data_provider = AnalysisDataProvider() diff --git a/src/py/crankshaft/crankshaft/random_seeds.py b/src/py/crankshaft/crankshaft/random_seeds.py index 31958cb..c55ba14 100644 --- a/src/py/crankshaft/crankshaft/random_seeds.py +++ b/src/py/crankshaft/crankshaft/random_seeds.py @@ -2,6 +2,7 @@ import random import numpy + def set_random_seeds(value): """ Set the seeds of the RNGs (Random Number Generators) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 3ad8273..c830bbc 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -11,7 +11,7 @@ import crankshaft.pysal_utils as pu from crankshaft.analysis_data_provider import AnalysisDataProvider -class Markov: +class Markov(object): def __init__(self, data_provider=None): if data_provider is None: self.data_provider = AnalysisDataProvider() From 4b3481b1a6571ed5f3aac04b9c0336efde46f098 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 12 Jan 2017 17:03:01 -0500 Subject: [PATCH 08/14] adds decorators to reduce boilerplate code --- .../crankshaft/analysis_data_provider.py | 67 +++++++------------ 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 25e30fc..9bed024 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -6,56 +6,45 @@ NULL_VALUE_ERROR = ('No usable data passed to analysis. Check your input rows ' 'for null values and fill in appropriately.') -def verify_data(data): - if len(data) == 0: - plpy.error(NULL_VALUE_ERROR) +def verify_data(f): + def wrapper(*args, **kwargs): + try: + print('kwargs: %s' % str(kwargs)) + data = f(*args, **kwargs) + if len(data) == 0: + plpy.error(NULL_VALUE_ERROR) + else: + return data + except Exception, err: + plpy.error('Analysis failed: {}'.format(err)) + return wrapper class AnalysisDataProvider(object): + @verify_data def get_getis(self, w_type, params): """fetch data for getis ord's g""" - try: - query = pu.construct_neighbor_query(w_type, params) - data = plpy.execute(query) - - # if there are no neighbors or all nulls, exit - verify_data(data) - return data - except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % err) + query = pu.construct_neighbor_query(w_type, params) + return plpy.execute(query) + @verify_data def get_markov(self, w_type, params): """fetch data for spatial markov""" - try: - query = pu.construct_neighbor_query(w_type, params) - data = plpy.execute(query) - - verify_data(data) - return data - except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % err) + query = pu.construct_neighbor_query(w_type, params) + return plpy.execute(query) + @verify_data def get_moran(self, w_type, params): """fetch data for moran's i analyses""" - try: - query = pu.construct_neighbor_query(w_type, params) - data = plpy.execute(query) - - # if there are no neighbors, exit - verify_data(data) - return data - except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % err) + query = pu.construct_neighbor_query(w_type, params) + return plpy.execute(query) + @verify_data def get_nonspatial_kmeans(self, query): """fetch data for non-spatial kmeans""" - try: - data = plpy.execute(query) - verify_data(data) - return data - except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % err) + return plpy.execute(query) + @verify_data def get_spatial_kmeans(self, params): """fetch data for spatial kmeans""" query = ''' @@ -66,9 +55,5 @@ class AnalysisDataProvider(object): FROM ({subquery}) As a WHERE "{geom_col}" IS NOT NULL '''.format(**params) - try: - data = plpy.execute(query) - verify_data(data) - return data - except plpy.SPIError, err: - plpy.error('Analysis failed: %s' % err) + + return plpy.execute(query) From 04bd067045fac24d893c5ea44e969edbf4c52f61 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 12 Jan 2017 17:12:09 -0500 Subject: [PATCH 09/14] standardizing naming conventions in code --- src/py/crankshaft/crankshaft/clustering/getis.py | 12 ++++++------ src/py/crankshaft/crankshaft/clustering/kmeans.py | 8 ++++---- .../crankshaft/space_time_dynamics/markov.py | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py index f560e9c..2bee3a2 100644 --- a/src/py/crankshaft/crankshaft/clustering/getis.py +++ b/src/py/crankshaft/crankshaft/clustering/getis.py @@ -31,13 +31,13 @@ class Getis(object): # geometries with attributes that are null are ignored # resulting in a collection of not as near neighbors if kNN is chosen - qvals = OrderedDict([("id_col", id_col), - ("attr1", attr), - ("geom_col", geom_col), - ("subquery", subquery), - ("num_ngbrs", num_ngbrs)]) + params = OrderedDict([("id_col", id_col), + ("attr1", attr), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) - result = self.data_provider.get_getis(w_type, qvals) + result = self.data_provider.get_getis(w_type, params) attr_vals = pu.get_attributes(result) # build PySAL weight object diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py index 6c1115a..094d47b 100644 --- a/src/py/crankshaft/crankshaft/clustering/kmeans.py +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -20,12 +20,12 @@ class Kmeans(object): "geom_col": "the_geom", "id_col": "cartodb_id"} - data = self.data_provider.get_spatial_kmeans(params) + result = self.data_provider.get_spatial_kmeans(params) # Unpack query response - xs = data[0]['xs'] - ys = data[0]['ys'] - ids = data[0]['ids'] + xs = result[0]['xs'] + ys = result[0]['ys'] + ids = result[0]['ids'] km = KMeans(n_clusters=no_clusters, n_init=no_init) labels = km.fit_predict(zip(xs, ys)) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index c830bbc..20daaf1 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -61,14 +61,14 @@ class Markov(object): "subquery": subquery, "num_ngbrs": num_ngbrs} - query_result = self.data_provider.get_markov(w_type, params) + result = self.data_provider.get_markov(w_type, params) # build weight - weights = pu.get_weight(query_result, w_type) + weights = pu.get_weight(result, w_type) weights.transform = 'r' # prep time data - t_data = get_time_data(query_result, time_cols) + t_data = get_time_data(result, time_cols) sp_markov_result = ps.Spatial_Markov(t_data, weights, From ddd69bb457a5cb7c92df0826e20f6bece7f3f7e2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 12 Jan 2017 17:12:40 -0500 Subject: [PATCH 10/14] adds mock error function --- src/py/crankshaft/test/mock_plpy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py index e8a279d..9c3340c 100644 --- a/src/py/crankshaft/test/mock_plpy.py +++ b/src/py/crankshaft/test/mock_plpy.py @@ -42,6 +42,9 @@ class MockPlPy: def info(self, msg): self.infos.append(msg) + def error(self, msg): + self.notices.append(msg) + def cursor(self, query): data = self.execute(query) return MockCursor(data) From be2bf19c0a958a1d322625b65acfbfb2ac779767 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 12 Jan 2017 17:14:32 -0500 Subject: [PATCH 11/14] removes print line --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 9bed024..bfc97ed 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -9,7 +9,6 @@ NULL_VALUE_ERROR = ('No usable data passed to analysis. Check your input rows ' def verify_data(f): def wrapper(*args, **kwargs): try: - print('kwargs: %s' % str(kwargs)) data = f(*args, **kwargs) if len(data) == 0: plpy.error(NULL_VALUE_ERROR) From 8e4bbb8a90cdd0c2ca3630bd6793d5f01dde4d71 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 13 Jan 2017 14:07:20 -0500 Subject: [PATCH 12/14] add default return value on verify_data wrapper --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index bfc97ed..8649773 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -16,6 +16,9 @@ def verify_data(f): return data except Exception, err: plpy.error('Analysis failed: {}'.format(err)) + + return [] + return wrapper From 77e73dbc75e110b106d1900670241d7671829f24 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 9 Jan 2018 10:23:38 -0500 Subject: [PATCH 13/14] updates error syntax --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 8649773..4a2bb67 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -14,7 +14,7 @@ def verify_data(f): plpy.error(NULL_VALUE_ERROR) else: return data - except Exception, err: + except Exception as err: plpy.error('Analysis failed: {}'.format(err)) return [] From 32bb3b12769f597a24795aff0a4540e5055c8391 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 9 Jan 2018 11:37:27 -0500 Subject: [PATCH 14/14] adds missing decorator for gwr_predict --- src/py/crankshaft/crankshaft/analysis_data_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 0ec4e9b..12737bf 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -59,7 +59,6 @@ class AnalysisDataProvider(object): FROM ({subquery}) As a WHERE "{geom_col}" IS NOT NULL '''.format(**params) - return plpy.execute(query) @verify_data @@ -68,6 +67,7 @@ class AnalysisDataProvider(object): query = pu.gwr_query(params) return plpy.execute(query) + @verify_data def get_gwr_predict(self, params): """fetch data for gwr predict""" query = pu.gwr_predict_query(params)