cartodb4/crankshaft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #122 from CartoDB/moran-query-ordering-fix

Browse Source 
Moran query ordering fix
This commit is contained in:
Mario de Frutos
2016-12-01 10:41:19 +01:00
committed by GitHub
parent 62076bb48c 803816f5c9
commit 25e453a882
3 changed files with 180 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
src/pg/test/expected/02_moran_test.out
View File

@@ -149,135 +149,135 @@ _cdb_random_seeds
(1 row)
code|quads
01|LL
02|LH
03|HH
04|HH
05|LL
06|HH
07|LL
08|LL
09|LL
10|HH
11|HH
12|HL
13|LL
14|HH
01|HH
02|HL
03|LL
04|LL
05|LH
06|LL
07|HH
08|HH
09|HH
10|LL
11|LL
12|LL
13|HL
14|LL
15|LL
16|LL
17|LL
18|LH
19|LL
20|LL
21|HH
22|LL
23|HL
16|HH
17|HH
18|LL
19|HH
20|HH
21|LL
22|HH
23|LL
24|LL
25|LL
26|LL
25|HH
26|HH
27|LL
28|LL
29|LH
30|HH
31|LL
28|HH
29|LL
30|LL
31|HH
32|LL
33|LL
34|LL
35|LH
36|HL
37|LH
38|LH
39|LL
40|LL
41|LH
42|HL
43|LL
44|HL
45|LL
46|HL
33|HL
34|LH
35|LL
36|LL
37|HL
38|HL
39|HH
40|HH
41|HL
42|LH
43|LH
44|LL
45|LH
46|LL
47|LL
48|LL
49|HL
50|LL
51|HH
(51 rows)
48|HH
49|LH
50|HH
51|LL
52|LL
(52 rows)
_cdb_random_seeds
(1 row)
code|quads
03|HH
04|HH
06|HH
10|HH
11|HH
12|HL
14|HH
21|HH
23|HL
30|HH
36|HL
42|HL
44|HL
46|HL
49|HL
51|HH
(16 rows)
01|HH
02|HL
07|HH
08|HH
09|HH
13|HL
16|HH
17|HH
19|HH
20|HH
22|HH
25|HH
26|HH
28|HH
31|HH
33|HL
37|HL
38|HL
39|HH
40|HH
41|HL
48|HH
50|HH
(23 rows)
_cdb_random_seeds
(1 row)
code|quads
01|LL
02|LH
05|LL
07|LL
08|LL
09|LL
13|LL
03|LL
04|LL
05|LH
06|LL
10|LL
11|LL
12|LL
14|LL
15|LL
16|LL
17|LL
18|LH
19|LL
20|LL
22|LL
18|LL
21|LL
23|LL
24|LL
25|LL
26|LL
27|LL
28|LL
29|LH
31|LL
29|LL
30|LL
32|LL
33|LL
34|LL
35|LH
37|LH
38|LH
39|LL
40|LL
41|LH
43|LL
45|LL
34|LH
35|LL
36|LL
42|LH
43|LH
44|LL
45|LH
46|LL
47|LL
48|LL
50|LL
(35 rows)
49|LH
51|LL
52|LL
(29 rows)
_cdb_random_seeds
(1 row)
code|quads
02|LH
12|HL
18|LH
23|HL
29|LH
35|LH
36|HL
37|LH
38|LH
41|LH
42|HL
44|HL
46|HL
49|HL
(14 rows)
02|HL
05|LH
13|HL
33|HL
34|LH
37|HL
38|HL
41|HL
42|LH
43|LH
45|LH
49|LH
(12 rows)

26
src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py
View File

@@ -45,8 +45,16 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5):
def query_attr_select(params):
"""
Create portion of SELECT statement for attributes inolved in query.
Defaults to order in the params
@param params: dict of information used in query (column names,
table name, etc.)
Example:
OrderedDict([('numerator', 'price'),
('denominator', 'sq_meters'),
('subquery', 'SELECT * FROM interesting_data')])
Output:
"i.\"price\"::numeric As attr1, " \
"i.\"sq_meters\"::numeric As attr2, "
"""
attr_string = ""
@@ -64,7 +72,7 @@ def query_attr_select(params):
if k not in ('id_col', 'geom_col', 'subquery',
'num_ngbrs', 'subquery')]
for idx, val in enumerate(sorted(attrs)):
for idx, val in enumerate(attrs):
attr_string += template % {"col": params[val],
"alias_num": idx + 1}
@@ -80,8 +88,8 @@ def query_attr_where(params):
'numerator': 'data1',
'denominator': 'data2',
'': ...}
Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2"
IS NOT NULL'
Output:
'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL'
Input:
{'subquery': ...,
'time_cols': ['time1', 'time2', 'time3'],
@@ -102,15 +110,17 @@ def query_attr_where(params):
# moran where clauses
# get keys
attrs = sorted([k for k in params
if k not in ('id_col', 'geom_col', 'subquery',
'num_ngbrs', 'subquery')])
attrs = [k for k in params
if k not in ('id_col', 'geom_col', 'subquery',
'num_ngbrs', 'subquery')]
# add values to template
for attr in attrs:
attr_string.append(template % params[attr])
if len(attrs) == 2:
attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]])
if 'denominator' in attrs:
attr_string.append(
"idx_replace.\"%s\" <> 0" % params['denominator'])
out = " AND ".join(attr_string)

86
src/py/crankshaft/test/test_pysal_utils.py
View File

@@ -2,18 +2,33 @@ import unittest
import crankshaft.pysal_utils as pu
from crankshaft import random_seeds
from collections import OrderedDict
class PysalUtilsTest(unittest.TestCase):
"""Testing class for utility functions related to PySAL integrations"""
def setUp(self):
self.params = {"id_col": "cartodb_id",
"attr1": "andy",
"attr2": "jay_z",
"subquery": "SELECT * FROM a_list",
"geom_col": "the_geom",
"num_ngbrs": 321}
self.params1 = OrderedDict([("id_col", "cartodb_id"),
("attr1", "andy"),
("attr2", "jay_z"),
("subquery", "SELECT * FROM a_list"),
("geom_col", "the_geom"),
("num_ngbrs", 321)])
self.params2 = OrderedDict([("id_col", "cartodb_id"),
("numerator", "price"),
("denominator", "sq_meters"),
("subquery", "SELECT * FROM pecan"),
("geom_col", "the_geom"),
("num_ngbrs", 321)])
self.params3 = OrderedDict([("id_col", "cartodb_id"),
("numerator", "sq_meters"),
("denominator", "price"),
("subquery", "SELECT * FROM pecan"),
("geom_col", "the_geom"),
("num_ngbrs", 321)])
self.params_array = {"id_col": "cartodb_id",
"time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"],
@@ -24,34 +39,41 @@ class PysalUtilsTest(unittest.TestCase):
def test_query_attr_select(self):
"""Test query_attr_select"""
ans = "i.\"andy\"::numeric As attr1, " \
"i.\"jay_z\"::numeric As attr2, "
ans1 = ("i.\"andy\"::numeric As attr1, "
"i.\"jay_z\"::numeric As attr2, ")
ans_array = "i.\"_2013_dec\"::numeric As attr1, " \
"i.\"_2014_jan\"::numeric As attr2, " \
"i.\"_2014_feb\"::numeric As attr3, "
ans2 = ("i.\"price\"::numeric As attr1, "
"i.\"sq_meters\"::numeric As attr2, ")
self.assertEqual(pu.query_attr_select(self.params), ans)
ans3 = ("i.\"sq_meters\"::numeric As attr1, "
"i.\"price\"::numeric As attr2, ")
ans_array = ("i.\"_2013_dec\"::numeric As attr1, "
"i.\"_2014_jan\"::numeric As attr2, "
"i.\"_2014_feb\"::numeric As attr3, ")
self.assertEqual(pu.query_attr_select(self.params1), ans1)
self.assertEqual(pu.query_attr_select(self.params2), ans2)
self.assertEqual(pu.query_attr_select(self.params3), ans3)
self.assertEqual(pu.query_attr_select(self.params_array), ans_array)
def test_query_attr_where(self):
"""Test pu.query_attr_where"""
ans = "idx_replace.\"andy\" IS NOT NULL AND " \
"idx_replace.\"jay_z\" IS NOT NULL AND " \
"idx_replace.\"jay_z\" <> 0"
ans1 = ("idx_replace.\"andy\" IS NOT NULL AND "
"idx_replace.\"jay_z\" IS NOT NULL")
ans_array = "idx_replace.\"_2013_dec\" IS NOT NULL AND " \
"idx_replace.\"_2014_jan\" IS NOT NULL AND " \
"idx_replace.\"_2014_feb\" IS NOT NULL"
ans_array = ("idx_replace.\"_2013_dec\" IS NOT NULL AND "
"idx_replace.\"_2014_jan\" IS NOT NULL AND "
"idx_replace.\"_2014_feb\" IS NOT NULL")
self.assertEqual(pu.query_attr_where(self.params), ans)
self.assertEqual(pu.query_attr_where(self.params1), ans1)
self.assertEqual(pu.query_attr_where(self.params_array), ans_array)
def test_knn(self):
"""Test knn neighbors constructor"""
ans = "SELECT i.\"cartodb_id\" As id, " \
ans1 = "SELECT i.\"cartodb_id\" As id, " \
"i.\"andy\"::numeric As attr1, " \
"i.\"jay_z\"::numeric As attr2, " \
"(SELECT ARRAY(SELECT j.\"cartodb_id\" " \
@@ -59,17 +81,15 @@ class PysalUtilsTest(unittest.TestCase):
"WHERE " \
"i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \
"j.\"andy\" IS NOT NULL AND " \
"j.\"jay_z\" IS NOT NULL AND " \
"j.\"jay_z\" <> 0 " \
"j.\"jay_z\" IS NOT NULL " \
"ORDER BY " \
"j.\"the_geom\" <-> i.\"the_geom\" ASC " \
"LIMIT 321)) As neighbors " \
"FROM (SELECT * FROM a_list) As i " \
"WHERE i.\"andy\" IS NOT NULL AND " \
"i.\"jay_z\" IS NOT NULL AND " \
"i.\"jay_z\" <> 0 " \
"i.\"jay_z\" IS NOT NULL " \
"ORDER BY i.\"cartodb_id\" ASC;"
ans_array = "SELECT i.\"cartodb_id\" As id, " \
"i.\"_2013_dec\"::numeric As attr1, " \
"i.\"_2014_jan\"::numeric As attr2, " \
@@ -88,13 +108,13 @@ class PysalUtilsTest(unittest.TestCase):
"i.\"_2014_feb\" IS NOT NULL "\
"ORDER BY i.\"cartodb_id\" ASC;"
self.assertEqual(pu.knn(self.params), ans)
self.assertEqual(pu.knn(self.params1), ans1)
self.assertEqual(pu.knn(self.params_array), ans_array)
def test_queen(self):
"""Test queen neighbors constructor"""
ans = "SELECT i.\"cartodb_id\" As id, " \
ans1 = "SELECT i.\"cartodb_id\" As id, " \
"i.\"andy\"::numeric As attr1, " \
"i.\"jay_z\"::numeric As attr2, " \
"(SELECT ARRAY(SELECT j.\"cartodb_id\" " \
@@ -104,23 +124,21 @@ class PysalUtilsTest(unittest.TestCase):
"ST_Touches(i.\"the_geom\", " \
"j.\"the_geom\") AND " \
"j.\"andy\" IS NOT NULL AND " \
"j.\"jay_z\" IS NOT NULL AND " \
"j.\"jay_z\" <> 0)" \
"j.\"jay_z\" IS NOT NULL)" \
") As neighbors " \
"FROM (SELECT * FROM a_list) As i " \
"WHERE i.\"andy\" IS NOT NULL AND " \
"i.\"jay_z\" IS NOT NULL AND " \
"i.\"jay_z\" <> 0 " \
"i.\"jay_z\" IS NOT NULL " \
"ORDER BY i.\"cartodb_id\" ASC;"
self.assertEqual(pu.queen(self.params), ans)
self.assertEqual(pu.queen(self.params1), ans1)
def test_construct_neighbor_query(self):
"""Test construct_neighbor_query"""
# Compare to raw knn query
self.assertEqual(pu.construct_neighbor_query('knn', self.params),
pu.knn(self.params))
self.assertEqual(pu.construct_neighbor_query('knn', self.params1),
pu.knn(self.params1))
def test_get_attributes(self):
"""Test get_attributes"""