Merge remote-tracking branch 'origin/develop' into segmentation

Conflicts:
	src/py/crankshaft/crankshaft/__init__.py

src/pg/sql/11_markov.sql

@@ -0,0 +1,81 @@
+-- Spatial Markov
+
+-- input table format:
+-- id | geom | date_1 | date_2 | date_3
+--  1 | Pt1  | 12.3   | 13.1   | 14.2
+--  2 | Pt2  | 11.0   | 13.2   | 12.5
+--  ...
+-- Sample Function call:
+-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate',
+--                          Array['date_1', 'date_2', 'date_3'])
+
+CREATE OR REPLACE FUNCTION
+  CDB_SpatialMarkovTrend (
+      subquery TEXT,
+      time_cols TEXT[],
+      num_classes INT DEFAULT 7,
+      w_type TEXT DEFAULT 'knn',
+      num_ngbrs INT DEFAULT 5,
+  	  permutations INT DEFAULT 99,
+  	  geom_col TEXT DEFAULT 'the_geom',
+  	  id_col TEXT DEFAULT 'cartodb_id')
+RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT)
+AS $$
+
+  from crankshaft.space_time_dynamics import spatial_markov_trend
+
+  ## TODO: use named parameters or a dictionary
+  return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
+$$ LANGUAGE plpythonu;
+
+-- input table format: identical to above but in a predictable format
+-- Sample function call:
+-- SELECT cdb_spatial_markov('SELECT * FROM real_estate',
+--                           'date_1')
+
+
+-- CREATE OR REPLACE FUNCTION
+--   cdb_spatial_markov (
+--       subquery TEXT,
+--       time_col_min text,
+--       time_col_max text,
+--       date_format text, -- '_YYYY_MM_DD'
+--       num_time_per_bin INT DEFAULT 1,
+--   	  permutations INT DEFAULT 99,
+--   	  geom_column TEXT DEFAULT 'the_geom',
+--   	  id_col TEXT DEFAULT 'cartodb_id',
+--       w_type TEXT DEFAULT 'knn',
+--       num_ngbrs int DEFAULT 5)
+-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT)
+-- AS $$
+--   plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
+--   from crankshaft.clustering import moran_local
+--   # TODO: use named parameters or a dictionary
+--   return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
+-- $$ LANGUAGE plpythonu;
+--
+-- -- input table format:
+-- -- id | geom | date  | measurement
+-- --  1 | Pt1  | 12/3  | 13.2
+-- --  2 | Pt2  | 11/5  | 11.3
+-- --  3 | Pt1  | 11/13 | 12.9
+-- --  4 | Pt3  | 12/19 | 10.1
+-- -- ...
+--
+-- CREATE OR REPLACE FUNCTION
+--   cdb_spatial_markov (
+--       subquery TEXT,
+--       time_col text,
+--       num_time_per_bin INT DEFAULT 1,
+--   	  permutations INT DEFAULT 99,
+--   	  geom_column TEXT DEFAULT 'the_geom',
+--   	  id_col TEXT DEFAULT 'cartodb_id',
+--       w_type TEXT DEFAULT 'knn',
+--       num_ngbrs int DEFAULT 5)
+-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT)
+-- AS $$
+--   plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
+--   from crankshaft.clustering import moran_local
+--   # TODO: use named parameters or a dictionary
+--   return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
+-- $$ LANGUAGE plpythonu;

src/pg/test/expected/05_markov_test.out

@@ -0,0 +1,10 @@
+SET client_min_messages TO WARNING;
+\set ECHO none
+_cdb_random_seeds
+
+(1 row)
+cartodb_id|trend_test|trend_up_test|trend_down_test|volatility_test
+1|t|t|t|t
+2|t|t|t|t
+3|t|t|t|t
+(3 rows)

src/pg/test/sql/05_markov_test.sql

@@ -0,0 +1,33 @@
+SET client_min_messages TO WARNING;
+\set ECHO none
+\pset format unaligned
+\i test/fixtures/markov_usjoin_example.sql
+
+-- Areas of Interest functions perform some nondeterministic computations
+-- (to estimate the significance); we will set the seeds for the RNGs
+-- that affect those results to have repeatable results
+SELECT cdb_crankshaft._cdb_random_seeds(1234);
+
+SELECT
+  m1.cartodb_id,
+  CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05
+       WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 < 0.05
+       WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05
+       ELSE NULL END As trend_test,
+  CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05
+       WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.13157894736842105) / 0.13157894736842105 < 0.05
+       WHEN m1.cartodb_id = 3 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05
+       ELSE NULL END As trend_up_test,
+  CASE WHEN m1.cartodb_id = 1 THEN m2.trend_down = 0.0
+       WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0
+       WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0
+       ELSE NULL END As trend_down_test,
+  CASE WHEN m1.cartodb_id = 1 THEN abs(m2.volatility - 0.367574633389) / 0.367574633389  < 0.1
+       WHEN m1.cartodb_id = 2 THEN abs(m2.volatility - 0.33807340742635211) / 0.33807340742635211  < 0.1
+       WHEN m1.cartodb_id = 3 THEN abs(m2.volatility - 0.3682585596149513) / 0.3682585596149513  < 0.1
+       ELSE NULL END As volatility_test
+  FROM markov_usjoin_example As m1
+  JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2
+    ON m1.cartodb_id = m2.rowid
+ORDER BY m1.cartodb_id
+LIMIT 3;