cartodb4/crankshaft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:CartoDB/crankshaft into add-salesforce

Browse Source
This commit is contained in:
abelvm
2016-09-22 15:09:54 +02:00
parent 9b1a889133 c7bb9698a9
commit d51b02b61e
71 changed files with 11587 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pg/crankshaft.control
+1 -1
View File
@@ -1,5 +1,5 @@
comment = 'CartoDB Spatial Analysis extension'
default_version = '0.4.0'
default_version = '0.4.2'
requires = 'plpythonu, postgis'
superuser = true
schema = cdb_crankshaft
src/pg/sql/08_interpolation.sql
+19 -5
View File
@@ -1,6 +1,8 @@
-- 0: nearest neighbor
-- 0: nearest neighbor(s)
-- 1: barymetric
-- 2: IDW
-- 3: krigin ---> TO DO
CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation(
IN query text,
@@ -50,12 +52,19 @@ DECLARE
vc numeric;
output numeric;
BEGIN
output := -999.999;
-- nearest
-- output := -999.999;
-- nearest neighbors
-- p1: limit the number of neighbors, 0-> closest one
IF method = 0 THEN
WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v)
SELECT a.v INTO output FROM a ORDER BY point<->a.g LIMIT 1;
IF p1 = 0 THEN
p1 := 1;
END IF;
WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v),
b as (SELECT a.v as v FROM a ORDER BY point<->a.g LIMIT p1::integer)
SELECT avg(b.v) INTO output FROM b;
RETURN output;
-- barymetric
@@ -121,6 +130,11 @@ BEGIN
SELECT sum(b.f)/sum(b.k) INTO output FROM b;
RETURN output;
-- krigin
ELSIF method = 3 THEN
-- TO DO
END IF;
RETURN -777.777;
src/pg/sql/10_moran.sql
+1 -1
View File
@@ -10,7 +10,7 @@ CREATE OR REPLACE FUNCTION
id_col TEXT DEFAULT 'cartodb_id')
RETURNS TABLE (moran NUMERIC, significance NUMERIC)
AS $$
from crankshaft.clustering import moran_local
from crankshaft.clustering import moran
# TODO: use named parameters or a dictionary
return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu;
src/pg/sql/19_contour.sql
+15 -12
View File
@@ -17,16 +17,15 @@ RETURNS TABLE(
DECLARE
cell_count integer;
tin geometry[];
resolution integer;
BEGIN
-- calc the cell size in web mercator units
-- WITH center as (
-- SELECT ST_centroid(ST_Collect(geomin)) as c
-- )
-- SELECT
-- round(resolution / cos(ST_y(c) * pi()/180))
-- INTO cell
-- FROM center;
-- raise notice 'Resol: %', cell;
-- nasty trick to override issue #121
IF max_time = 0 THEN
max_time = -90;
END IF;
resolution := max_time;
max_time := -1 * resolution;
-- calc the optimal number of cells for the current dataset
SELECT
@@ -70,9 +69,13 @@ BEGIN
),
resolution as(
SELECT
round(|/ (
ST_area(geom) / cell_count
)) as cell
CASE WHEN resolution <= 0 THEN
round(|/ (
ST_area(geom) / abs(cell_count)
))
ELSE
resolution
END AS cell
FROM envelope3857
),
grid as(
src/pg/test/expected/02_moran_test.out
+6
View File
@@ -5,6 +5,12 @@ SET client_min_messages TO WARNING;
\set ECHO none
_cdb_random_seeds
(1 row)
moran|significance
0.3399|-0.0196
(1 row)
_cdb_random_seeds
(1 row)
code|quads
01|HH
src/pg/test/expected/08_interpolation_test.out
+3 -3
View File
@@ -1,7 +1,7 @@
SET client_min_messages TO WARNING;
\set ECHO none
nn | nni | idw
-----+--------------------------+-----------------
200 | 238.41059602632179224595 | 341.46260750526
nn | nni | idw
----------------------+--------------------------+-----------------
200.0000000000000000 | 238.41059602632179224595 | 341.46260750526
(1 row)
src/pg/test/sql/02_moran_test.sql
+8
View File
@@ -6,6 +6,14 @@
-- Areas of Interest functions perform some nondeterministic computations
-- (to estimate the significance); we will set the seeds for the RNGs
-- that affect those results to have repeateble results
-- Moran's I Global
SELECT cdb_crankshaft._cdb_random_seeds(1234);
SELECT round(moran, 4) As moran, round(significance, 4) As significance
FROM cdb_crankshaft.CDB_AreasOfInterestGlobal('SELECT * FROM ppoints', 'value') m(moran, significance);
-- Moran's I Local
SELECT cdb_crankshaft._cdb_random_seeds(1234);
SELECT ppoints.code, m.quads
src/pg/test/sql/19_contour_test.sql
+1 -1
View File
@@ -12,6 +12,6 @@ SELECT
foo.*
FROM
a,
cdb_crankshaft.CDB_contour(a.g, a.vals, 0.0, 1, 3, 5, 60) foo
cdb_crankshaft.CDB_contour(a.g, a.vals, 0.0, 1, 3, 5, -60) foo
)
SELECT bin, avg_value from b order by bin;
src/py/crankshaft/crankshaft/clustering/moran.py
+23 -35
View File
@@ -14,6 +14,7 @@ import crankshaft.pysal_utils as pu
# High level interface ---------------------------------------
def moran(subquery, attr_name,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
@@ -30,32 +31,28 @@ def moran(subquery, attr_name,
query = pu.construct_neighbor_query(w_type, qvals)
plpy.notice('** Query: %s' % query)
try:
result = plpy.execute(query)
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(2)
plpy.notice('** Query returned with %d rows' % len(result))
except plpy.SPIError:
plpy.error('Error: areas of interest query failed, check input parameters')
plpy.notice('** Query failed: "%s"' % query)
plpy.notice('** Error: %s' % plpy.SPIError)
except plpy.SPIError, e:
plpy.error('Analysis failed: %s' % e)
return pu.empty_zipped_array(2)
## collect attributes
# collect attributes
attr_vals = pu.get_attributes(result)
## calculate weights
# calculate weights
weight = pu.get_weight(result, w_type, num_ngbrs)
## calculate moran global
# calculate moran global
moran_global = ps.esda.moran.Moran(attr_vals, weight,
permutations=permutations)
return zip([moran_global.I], [moran_global.EI])
def moran_local(subquery, attr,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
@@ -79,9 +76,8 @@ def moran_local(subquery, attr,
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(5)
except plpy.SPIError:
plpy.error('Error: areas of interest query failed, check input parameters')
plpy.notice('** Query failed: "%s"' % query)
except plpy.SPIError, e:
plpy.error('Analysis failed: %s' % e)
return pu.empty_zipped_array(5)
attr_vals = pu.get_attributes(result)
@@ -96,6 +92,7 @@ def moran_local(subquery, attr,
return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
def moran_rate(subquery, numerator, denominator,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
@@ -111,32 +108,28 @@ def moran_rate(subquery, numerator, denominator,
query = pu.construct_neighbor_query(w_type, qvals)
plpy.notice('** Query: %s' % query)
try:
result = plpy.execute(query)
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(2)
plpy.notice('** Query returned with %d rows' % len(result))
except plpy.SPIError:
plpy.error('Error: areas of interest query failed, check input parameters')
plpy.notice('** Query failed: "%s"' % query)
plpy.notice('** Error: %s' % plpy.SPIError)
except plpy.SPIError, e:
plpy.error('Analysis failed: %s' % e)
return pu.empty_zipped_array(2)
## collect attributes
# collect attributes
numer = pu.get_attributes(result, 1)
denom = pu.get_attributes(result, 2)
weight = pu.get_weight(result, w_type, num_ngbrs)
## calculate moran global rate
# calculate moran global rate
lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight,
permutations=permutations)
return zip([lisa_rate.I], [lisa_rate.EI])
def moran_local_rate(subquery, numerator, denominator,
w_type, num_ngbrs, permutations, geom_col, id_col):
"""
@@ -160,13 +153,11 @@ def moran_local_rate(subquery, numerator, denominator,
# if there are no neighbors, exit
if len(result) == 0:
return pu.empty_zipped_array(5)
except plpy.SPIError:
plpy.error('Error: areas of interest query failed, check input parameters')
plpy.notice('** Query failed: "%s"' % query)
plpy.notice('** Error: %s' % plpy.SPIError)
except plpy.SPIError, e:
plpy.error('Analysis failed: %s' % e)
return pu.empty_zipped_array(5)
## collect attributes
# collect attributes
numer = pu.get_attributes(result, 1)
denom = pu.get_attributes(result, 2)
@@ -181,12 +172,12 @@ def moran_local_rate(subquery, numerator, denominator,
return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
def moran_local_bv(subquery, attr1, attr2,
permutations, geom_col, id_col, w_type, num_ngbrs):
"""
Moran's I (local) Bivariate (untested)
"""
plpy.notice('** Constructing query')
qvals = OrderedDict([("id_col", id_col),
("attr1", attr1),
@@ -203,12 +194,11 @@ def moran_local_bv(subquery, attr1, attr2,
if len(result) == 0:
return pu.empty_zipped_array(4)
except plpy.SPIError:
plpy.error("Error: areas of interest query failed, " \
plpy.error("Error: areas of interest query failed, "
"check input parameters")
plpy.notice('** Query failed: "%s"' % query)
return pu.empty_zipped_array(4)
## collect attributes
# collect attributes
attr1_vals = pu.get_attributes(result, 1)
attr2_vals = pu.get_attributes(result, 2)
@@ -219,17 +209,14 @@ def moran_local_bv(subquery, attr1, attr2,
lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight,
permutations=permutations)
plpy.notice("len of Is: %d" % len(lisa.Is))
# find clustering of significance
lisa_sig = quad_position(lisa.q)
plpy.notice('** Finished calculations')
return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
# Low level functions ----------------------------------------
def map_quads(coord):
"""
Map a quadrant number to Moran's I designation
@@ -250,6 +237,7 @@ def map_quads(coord):
else:
return None
def quad_position(quads):
"""
Produce Moran's I classification based of n
src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py
+25 -12
View File
@@ -6,6 +6,7 @@
import numpy as np
import pysal as ps
def construct_neighbor_query(w_type, query_vals):
"""Return query (a string) used for finding neighbors
@param w_type text: type of neighbors to calculate ('knn' or 'queen')
@@ -17,7 +18,8 @@ def construct_neighbor_query(w_type, query_vals):
else:
return queen(query_vals)
## Build weight object
# Build weight object
def get_weight(query_res, w_type='knn', num_ngbrs=5):
"""
Construct PySAL weight from return value of query
@@ -39,6 +41,7 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5):
return built_weight
def query_attr_select(params):
"""
Create portion of SELECT statement for attributes inolved in query.
@@ -50,21 +53,24 @@ def query_attr_select(params):
template = "i.\"%(col)s\"::numeric As attr%(alias_num)s, "
if 'time_cols' in params:
## if markov analysis
# if markov analysis
attrs = params['time_cols']
for idx, val in enumerate(attrs):
attr_string += template % {"col": val, "alias_num": idx + 1}
else:
## if moran's analysis
# if moran's analysis
attrs = [k for k in params
if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]
if k not in ('id_col', 'geom_col', 'subquery',
'num_ngbrs', 'subquery')]
for idx, val in enumerate(sorted(attrs)):
attr_string += template % {"col": params[val], "alias_num": idx + 1}
attr_string += template % {"col": params[val],
"alias_num": idx + 1}
return attr_string
def query_attr_where(params):
"""
Construct where conditions when building neighbors query
@@ -74,7 +80,8 @@ def query_attr_where(params):
'numerator': 'data1',
'denominator': 'data2',
'': ...}
Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL'
Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2"
IS NOT NULL'
Input:
{'subquery': ...,
'time_cols': ['time1', 'time2', 'time3'],
@@ -86,17 +93,18 @@ def query_attr_where(params):
template = "idx_replace.\"%s\" IS NOT NULL"
if 'time_cols' in params:
## markov where clauses
# markov where clauses
attrs = params['time_cols']
# add values to template
for attr in attrs:
attr_string.append(template % attr)
else:
## moran where clauses
# moran where clauses
# get keys
attrs = sorted([k for k in params
if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')])
if k not in ('id_col', 'geom_col', 'subquery',
'num_ngbrs', 'subquery')])
# add values to template
for attr in attrs:
attr_string.append(template % params[attr])
@@ -108,6 +116,7 @@ def query_attr_where(params):
return out
def knn(params):
"""SQL query for k-nearest neighbors.
@param vars: dict of values to fill template
@@ -139,7 +148,8 @@ def knn(params):
return query.format(**params)
## SQL query for finding queens neighbors (all contiguous polygons)
# SQL query for finding queens neighbors (all contiguous polygons)
def queen(params):
"""SQL query for queen neighbors.
@param params dict: information to fill query
@@ -167,14 +177,17 @@ def queen(params):
return query.format(**params)
## to add more weight methods open a ticket or pull request
# to add more weight methods open a ticket or pull request
def get_attributes(query_res, attr_num=1):
"""
@param query_res: query results with attributes and neighbors
@param attr_num: attribute number (1, 2, ...)
"""
return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float)
return np.array([x['attr' + str(attr_num)] for x in query_res],
dtype=np.float)
def empty_zipped_array(num_nones):
"""
src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+2 -2
View File
@@ -56,9 +56,9 @@ def spatial_markov_trend(subquery, time_cols, num_classes=7,
)
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, err:
except plpy.SPIError, e:
plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error('Query failed, check the input parameters')
plpy.error('Analysis failed: %s' % e)
return zip([None], [None], [None], [None], [None])
## build weight
src/py/crankshaft/test/test_clustering_moran.py
+33 -15
View File
@@ -14,6 +14,7 @@ import crankshaft.pysal_utils as pu
from crankshaft import random_seeds
import json
class MoranTest(unittest.TestCase):
"""Testing class for Moran's I functions"""
@@ -26,12 +27,15 @@ class MoranTest(unittest.TestCase):
"geom_col": "the_geom",
"num_ngbrs": 321}
self.params_markov = {"id_col": "cartodb_id",
"time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"],
"time_cols": ["_2013_dec", "_2014_jan",
"_2014_feb"],
"subquery": "SELECT * FROM a_list",
"geom_col": "the_geom",
"num_ngbrs": 321}
self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read())
self.moran_data = json.loads(open(fixture_file('moran.json')).read())
self.neighbors_data = json.loads(
open(fixture_file('neighbors.json')).read())
self.moran_data = json.loads(
open(fixture_file('moran.json')).read())
def test_map_quads(self):
"""Test map_quads"""
@@ -54,35 +58,49 @@ class MoranTest(unittest.TestCase):
def test_moran_local(self):
"""Test Moran's I local"""
data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data]
data = [{'id': d['id'],
'attr1': d['value'],
'neighbors': d['neighbors']} for d in self.neighbors_data]
plpy._define_result('select', data)
random_seeds.set_random_seeds(1234)
result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id')
result = cc.moran_local('subquery', 'value',
'knn', 5, 99, 'the_geom', 'cartodb_id')
result = [(row[0], row[1]) for row in result]
expected = self.moran_data
for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected):
zipped_values = zip(result, self.moran_data)
for ([res_val, res_quad], [exp_val, exp_quad]) in zipped_values:
self.assertAlmostEqual(res_val, exp_val)
self.assertEqual(res_quad, exp_quad)
def test_moran_local_rate(self):
"""Test Moran's I rate"""
data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data]
data = [{'id': d['id'],
'attr1': d['value'],
'attr2': 1,
'neighbors': d['neighbors']} for d in self.neighbors_data]
plpy._define_result('select', data)
random_seeds.set_random_seeds(1234)
result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id')
print 'result == None? ', result == None
result = cc.moran_local_rate('subquery', 'numerator', 'denominator',
'knn', 5, 99, 'the_geom', 'cartodb_id')
result = [(row[0], row[1]) for row in result]
expected = self.moran_data
for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected):
zipped_values = zip(result, self.moran_data)
for ([res_val, res_quad], [exp_val, exp_quad]) in zipped_values:
self.assertAlmostEqual(res_val, exp_val)
def test_moran(self):
"""Test Moran's I global"""
data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data]
data = [{'id': d['id'],
'attr1': d['value'],
'neighbors': d['neighbors']} for d in self.neighbors_data]
plpy._define_result('select', data)
random_seeds.set_random_seeds(1235)
result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id')
print 'result == None?', result == None
result = cc.moran('table', 'value',
'knn', 5, 99, 'the_geom', 'cartodb_id')
result_moran = result[0][0]
expected_moran = np.array([row[0] for row in self.moran_data]).mean()
self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2)