From 947d6ba798e5cc9f23d0b666be40b5073629c99b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 11 Oct 2016 16:38:18 -0400
Subject: [PATCH 01/40] first add

---
 src/pg/sql/11_kmeans.sql                      | 39 ++++++----
 .../crankshaft/clustering/kmeans.py           | 71 ++++++++++++++++---
 2 files changed, 86 insertions(+), 24 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index f20942f..4985c2f 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -1,21 +1,34 @@
-CREATE OR REPLACE FUNCTION  CDB_KMeans(query text, no_clusters integer,no_init integer default 20)
-RETURNS table (cartodb_id integer, cluster_no integer) as $$
-    
-    from crankshaft.clustering import kmeans
-    return kmeans(query,no_clusters,no_init)
+-- Spatial k-means clustering
 
-$$ language plpythonu;
+CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer, no_init integer default 20)
+RETURNS table (cartodb_id integer, cluster_no integer) as $$
+
+    from crankshaft.clustering import kmeans
+    return kmeans(query, no_clusters, no_init)
+
+$$ LANGUAGE plpythonu;
+
+-- Non-spatial k-means clustering
+-- query: sql query to retrieve all the needed data
+
+CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, col_names TEXT[], no_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
+RETURNS TABLE(rowid BIGINT, cluster_no INTEGER, )
+
+from crankshaft.clustering import kmeans_nonspatial
+return kmeans_nonspatial(query, colnames, num_clusters, id_col)
+
+$$ LANGUAGE plpythonu;
 
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)
-RETURNS Numeric[] AS 
+RETURNS Numeric[] AS
 $$
-DECLARE 
+DECLARE
     newX NUMERIC;
     newY NUMERIC;
     newW NUMERIC;
 BEGIN
-    IF weight IS NULL OR the_geom IS NULL THEN 
+    IF weight IS NULL OR the_geom IS NULL THEN
         newX = state[1];
         newY = state[2];
         newW = state[3];
@@ -30,12 +43,12 @@ END
 $$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[])
-RETURNS GEOMETRY AS 
+RETURNS GEOMETRY AS
 $$
 BEGIN
-    IF state[3] = 0 THEN 
+    IF state[3] = 0 THEN
         RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326);
-    ELSE 
+    ELSE
         RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326);
     END IF;
 END
@@ -56,7 +69,7 @@ BEGIN
             SFUNC = CDB_WeightedMeanS,
             FINALFUNC = CDB_WeightedMeanF,
             STYPE = Numeric[],
-            INITCOND = "{0.0,0.0,0.0}" 
+            INITCOND = "{0.0,0.0,0.0}"
         );
     END IF;
 END
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 4134062..ee2f304 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -1,18 +1,67 @@
 from sklearn.cluster import KMeans
 import plpy
 
-def kmeans(query, no_clusters, no_init=20):
-    data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids,
-        array_agg(ST_X(the_geom) order by cartodb_id) xs,
-        array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a
-        where the_geom is not null
-    '''.format(query=query))
 
-    xs  = data[0]['xs']
-    ys  = data[0]['ys']
+def kmeans(query, no_clusters, no_init=20):
+    """
+
+    """
+    full_query = '''
+      SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,
+             array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,
+             array_agg(ST_Y(the_geom) ORDER BY cartodb_id)
+        FROM ({query}) As a
+       WHERE the_geom IS NOT NULL
+        '''.format(query=query)
+    try:
+        data = plpy.execute(full_query)
+    except plpy.SPIError, err:
+        plpy.error("KMeans cluster failed: %s" % err)
+
+    xs = data[0]['xs']
+    ys = data[0]['ys']
     ids = data[0]['ids']
 
-    km = KMeans(n_clusters= no_clusters, n_init=no_init)
-    labels = km.fit_predict(zip(xs,ys))
-    return zip(ids,labels)
+    km = KMeans(n_clusters=no_clusters, n_init=no_init)
+    labels = km.fit_predict(zip(xs, ys))
+    return zip(ids, labels)
 
+
+def kmeans_nonspatial(query, colnames, num_clusters=5, id_col='cartodb_id'):
+    """
+        query (string): A SQL query to retrieve the data required to do the
+                        k-means clustering analysis, like so:
+                        SELECT * FROM iris_flower_data
+        colnames (list): a list of the column names which contain the data of
+                         interest, like so: ["sepal_width", "petal_width",
+                                             "sepal_length", "petal_length"]
+        num_clusters (int): number of clusters (greater than zero)
+        id_col (string): name of the input id_column
+    """
+
+    id_colname = 'rowids'
+
+    full_query = '''
+        SELECT {cols}, array_agg({id_col}) As {id_colname}
+        FROM ({query}) As a
+    '''.format(query=query,
+               id_col=id_col,
+               id_colname=id_colname,
+               cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
+                               for idx, val in enumerate(colnames)]))
+
+    try:
+        data = plpy.execute(full_query)
+        plpy.notice('query: %s' % full_query)
+
+        # fill array with values for kmeans clustering
+        data = np.array([d[c] for c in d if c != 'id_colname'],
+                        dtype=float).T
+    except plpy.SPIError, err:
+        plpy.error('KMeans cluster failed: %s' % err)
+
+    kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(data)
+
+    # zip(ids, labels, means)
+    return zip(kmeans.labels_, map(str, kmeans.cluster_centers_),
+               d[0]['rowids'])

From 3e1cef9958984e85ed5a6f859f7cb9374c50d9cb Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 11 Oct 2016 16:48:22 -0400
Subject: [PATCH 02/40] fix output signature

---
 src/pg/sql/11_kmeans.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 4985c2f..fe078b5 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -12,7 +12,7 @@ $$ LANGUAGE plpythonu;
 -- query: sql query to retrieve all the needed data
 
 CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, col_names TEXT[], no_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
-RETURNS TABLE(rowid BIGINT, cluster_no INTEGER, )
+RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint)
 
 from crankshaft.clustering import kmeans_nonspatial
 return kmeans_nonspatial(query, colnames, num_clusters, id_col)

From c47116571f0d6284dea600fbccb052b428909308 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Wed, 12 Oct 2016 14:19:19 -0400
Subject: [PATCH 03/40] properly close plpgsql function

---
 src/pg/sql/11_kmeans.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index fe078b5..59fcf59 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -12,7 +12,7 @@ $$ LANGUAGE plpythonu;
 -- query: sql query to retrieve all the needed data
 
 CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, col_names TEXT[], no_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
-RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint)
+RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint) AS $$
 
 from crankshaft.clustering import kmeans_nonspatial
 return kmeans_nonspatial(query, colnames, num_clusters, id_col)

From 361505fca9d7767886c9e290f545b49921337b29 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Wed, 12 Oct 2016 21:13:51 +0000
Subject: [PATCH 04/40] fixes syntax errors

---
 src/pg/sql/11_kmeans.sql                          |  2 +-
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 59fcf59..2db57e0 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -11,7 +11,7 @@ $$ LANGUAGE plpythonu;
 -- Non-spatial k-means clustering
 -- query: sql query to retrieve all the needed data
 
-CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, col_names TEXT[], no_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
+CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, colnames TEXT[], num_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint) AS $$
 
 from crankshaft.clustering import kmeans_nonspatial
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index ee2f304..091e87b 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -38,7 +38,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5, id_col='cartodb_id'):
         num_clusters (int): number of clusters (greater than zero)
         id_col (string): name of the input id_column
     """
-
+    import numpy as np
     id_colname = 'rowids'
 
     full_query = '''
@@ -55,13 +55,14 @@ def kmeans_nonspatial(query, colnames, num_clusters=5, id_col='cartodb_id'):
         plpy.notice('query: %s' % full_query)
 
         # fill array with values for kmeans clustering
-        data = np.array([d[c] for c in d if c != 'id_colname'],
-                        dtype=float).T
+        cluster_columns = np.array([data[0][c] for c in data.colnames() 
+                                    if c != 'id_colname'],
+                                   dtype=float).T
     except plpy.SPIError, err:
         plpy.error('KMeans cluster failed: %s' % err)
 
-    kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(data)
+    kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(cluster_columns)
 
     # zip(ids, labels, means)
     return zip(kmeans.labels_, map(str, kmeans.cluster_centers_),
-               d[0]['rowids'])
+               data[0]['rowids'])

From c2e2359e6520a71016bc15fe66613a67fbd526f4 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Wed, 12 Oct 2016 17:16:52 -0400
Subject: [PATCH 05/40] addes minmax scaling for variables

---
 .../crankshaft/clustering/kmeans.py           | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index ee2f304..aa2239c 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -38,6 +38,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5, id_col='cartodb_id'):
         num_clusters (int): number of clusters (greater than zero)
         id_col (string): name of the input id_column
     """
+    import numpy as np
 
     id_colname = 'rowids'
 
@@ -53,15 +54,26 @@ def kmeans_nonspatial(query, colnames, num_clusters=5, id_col='cartodb_id'):
     try:
         data = plpy.execute(full_query)
         plpy.notice('query: %s' % full_query)
-
-        # fill array with values for kmeans clustering
-        data = np.array([d[c] for c in d if c != 'id_colname'],
-                        dtype=float).T
     except plpy.SPIError, err:
         plpy.error('KMeans cluster failed: %s' % err)
 
+    # fill array with values for kmeans clustering
+    cluster_columns = scale_data(
+                        np.array([data[0][c] for c in data.colnames()
+                                  if c != id_col],
+                                 dtype=float).T)
+
     kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(data)
 
-    # zip(ids, labels, means)
     return zip(kmeans.labels_, map(str, kmeans.cluster_centers_),
-               d[0]['rowids'])
+               data[0]['rowids'])
+
+
+def scale_data(input_data):
+    """
+        Scale all input columns from 0 to 1 so that k-means puts them on equal
+        footing
+    """
+    from sklearn.preprocessing import MinMaxScaler
+    min_max_scaler = MinMaxScaler()
+    return min_max_scaler.fit_transform(input_data)

From a95423174cae1b1f3a0f30aca643a06a54a66e49 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 13 Oct 2016 10:50:48 -0400
Subject: [PATCH 06/40] adds back alias for kmeans removed by accident

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 1c1f178..c99ded1 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -9,7 +9,7 @@ def kmeans(query, no_clusters, no_init=20):
     full_query = '''
       SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,
              array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,
-             array_agg(ST_Y(the_geom) ORDER BY cartodb_id)
+             array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys
         FROM ({query}) As a
        WHERE the_geom IS NOT NULL
         '''.format(query=query)
@@ -87,8 +87,7 @@ def extract_columns(db_resp, id_col):
 
 def scale_data(features):
     """
-        Scale all input columns from 0 to 1 so that k-means puts them on equal
-        footing
+        Scale all input columns to center on 0 with a standard devation of 1
         input_data (numpy array): an array of dimension (n_features, n_samples)
     """
     from sklearn.preprocessing import StandardScaler

From 5d2a1881b1ed52b8bc4cc555762b60ceb6c16164 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 13 Oct 2016 15:00:28 +0000
Subject: [PATCH 07/40] make numpy with global scope in module

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index c99ded1..18a711f 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -1,6 +1,6 @@
 from sklearn.cluster import KMeans
 import plpy
-
+import numpy as np
 
 def kmeans(query, no_clusters, no_init=20):
     """
@@ -39,7 +39,6 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
         num_clusters (int): number of clusters (greater than zero)
         id_col (string): name of the input id_column
     """
-    import numpy as np
     out_id_colname = 'rowids'
     # TODO: need a random seed?
 

From 0feaf36cf62f8963b05014d50be477778e7e818f Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 13 Oct 2016 15:52:00 +0000
Subject: [PATCH 08/40] outputting consistent labels and centers

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 18a711f..86e8931 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -53,14 +53,13 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
 
     try:
         db_resp = plpy.execute(full_query)
-        plpy.notice('query: %s' % full_query)
     except plpy.SPIError, err:
         plpy.error('k-means cluster analysis failed: %s' % err)
 
     # fill array with values for kmeans clustering
     if standarize:
         cluster_columns = scale_data(
-          extract_columns(db_resp, id_col='cartodb_id'))
+          extract_columns(db_resp, id_col=out_id_colname))
     else:
         cluster_columns = extract_columns(db_resp)
 
@@ -69,7 +68,8 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
     kmeans = KMeans(n_clusters=num_clusters,
                     random_state=0).fit(cluster_columns)
 
-    return zip(kmeans.labels_, map(str, kmeans.cluster_centers_),
+    return zip(kmeans.labels_,
+               map(str, kmeans.cluster_centers_[kmeans.labels_]),
                db_resp[0][out_id_colname])
 
 

From b255fd3e0656ea8c47a26b16f1e6299329ae0057 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 13 Oct 2016 12:50:46 -0400
Subject: [PATCH 09/40] make private functions more explictly private

---
 .../crankshaft/crankshaft/clustering/kmeans.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index c99ded1..ac0ce4d 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -60,23 +60,27 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
 
     # fill array with values for kmeans clustering
     if standarize:
-        cluster_columns = scale_data(
-          extract_columns(db_resp, id_col='cartodb_id'))
+        cluster_columns = _scale_data(
+          _extract_columns(db_resp, id_col='cartodb_id'))
     else:
-        cluster_columns = extract_columns(db_resp)
+        cluster_columns = _extract_columns(db_resp)
 
     # TODO: decide on optimal parameters for most cases
     #       Are there ways of deciding parameters based on inputs?
     kmeans = KMeans(n_clusters=num_clusters,
                     random_state=0).fit(cluster_columns)
 
-    return zip(kmeans.labels_, map(str, kmeans.cluster_centers_),
+    return zip(kmeans.predict(X),
+               map(str, kmeans.cluster_centers_[kmeans.labels_]),
                db_resp[0][out_id_colname])
 
 
-def extract_columns(db_resp, id_col):
+def _extract_columns(db_resp, id_col):
     """
         Extract the features from the query and pack them into a NumPy array
+        db_resp (plpy data object): result of the kmeans request
+        id_col (string): name of column which has the row id (not a feature of
+                         the analysis)
     """
     return np.array([db_resp[0][c] for c in db_resp.colnames()
                      if c != id_col],
@@ -85,10 +89,10 @@ def extract_columns(db_resp, id_col):
 # -- Preprocessing steps
 
 
-def scale_data(features):
+def _scale_data(features):
     """
         Scale all input columns to center on 0 with a standard devation of 1
-        input_data (numpy array): an array of dimension (n_features, n_samples)
+        features (numpy array): an array of dimension (n_features, n_samples)
     """
     from sklearn.preprocessing import StandardScaler
     return StandardScaler().fit_transform(features)

From a370a2da52632c50611abe74c9094bccea8546ef Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 11:50:59 -0600
Subject: [PATCH 10/40] pep8 updates of test file

---
 src/py/crankshaft/test/test_cluster_kmeans.py | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/py/crankshaft/test/test_cluster_kmeans.py b/src/py/crankshaft/test/test_cluster_kmeans.py
index aba8e07..f1b738a 100644
--- a/src/py/crankshaft/test/test_cluster_kmeans.py
+++ b/src/py/crankshaft/test/test_cluster_kmeans.py
@@ -14,25 +14,25 @@ import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
 
+
 class KMeansTest(unittest.TestCase):
     """Testing class for Moran's I functions"""
 
     def setUp(self):
         plpy._reset()
-        self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read())
+        self.cluster_data = json.loads(
+          open(fixture_file('kmeans.json')).read())
         self.params = {"subquery": "select * from table",
-                       "no_clusters": "10"
-                       }
+                       "no_clusters": "10"}
 
     def test_kmeans(self):
         data = self.cluster_data
-        plpy._define_result('select' ,data)
+        plpy._define_result('select', data)
         clusters = cc.kmeans('subquery', 2)
-        labels  = [a[1] for a in clusters]
-        c1 = [a for a in clusters if a[1]==0]
-        c2 = [a for a in clusters if a[1]==1]
-
-        self.assertEqual(len(np.unique(labels)),2)
-        self.assertEqual(len(c1),20)
-        self.assertEqual(len(c2),20)
+        labels = [a[1] for a in clusters]
+        c1 = [a for a in clusters if a[1] == 0]
+        c2 = [a for a in clusters if a[1] == 1]
 
+        self.assertEqual(len(np.unique(labels)), 2)
+        self.assertEqual(len(c1), 20)
+        self.assertEqual(len(c2), 20)

From da23b002cfa62256dd2ebe26cb5d5638ba3998d6 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 11:51:53 -0600
Subject: [PATCH 11/40] rename to match submodule name

---
 .../test/{test_cluster_kmeans.py => test_clustering_kmeans.py}    | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/py/crankshaft/test/{test_cluster_kmeans.py => test_clustering_kmeans.py} (100%)

diff --git a/src/py/crankshaft/test/test_cluster_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
similarity index 100%
rename from src/py/crankshaft/test/test_cluster_kmeans.py
rename to src/py/crankshaft/test/test_clustering_kmeans.py

From 54bbd18b02bf74fa2f04e732088407745fb7affb Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 12:12:38 -0600
Subject: [PATCH 12/40] remove unneeded modules from test script

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index f1b738a..0d531ab 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -8,15 +8,12 @@ import numpy as np
 # import sys
 # sys.modules['plpy'] = plpy
 from helper import plpy, fixture_file
-import numpy as np
 import crankshaft.clustering as cc
-import crankshaft.pysal_utils as pu
-from crankshaft import random_seeds
 import json
 
 
 class KMeansTest(unittest.TestCase):
-    """Testing class for Moran's I functions"""
+    """Testing class for k-means spatial"""
 
     def setUp(self):
         plpy._reset()

From f800a35fd16ccab47e5417b9f6fdd8f277bba272 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 13:01:31 -0600
Subject: [PATCH 13/40] new format for input data

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 0d531ab..8ab300d 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -23,7 +23,12 @@ class KMeansTest(unittest.TestCase):
                        "no_clusters": "10"}
 
     def test_kmeans(self):
-        data = self.cluster_data
+        """
+        """
+        data = [{'xs': d['xs'],
+                 'ys': d['ys'],
+                 'id': d['id']} for d in self.cluster_data]
+
         plpy._define_result('select', data)
         clusters = cc.kmeans('subquery', 2)
         labels = [a[1] for a in clusters]

From f0c6cca76604c10952d45fad11d4610bf496c75b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 13:05:56 -0600
Subject: [PATCH 14/40] fix key name

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 8ab300d..e130da4 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -27,7 +27,7 @@ class KMeansTest(unittest.TestCase):
         """
         data = [{'xs': d['xs'],
                  'ys': d['ys'],
-                 'id': d['id']} for d in self.cluster_data]
+                 'ids': d['ids']} for d in self.cluster_data]
 
         plpy._define_result('select', data)
         clusters = cc.kmeans('subquery', 2)

From 5d8641732f81c444a92d470e6e34abe1ef51a0da Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 19:30:09 +0000
Subject: [PATCH 15/40] change string formatting

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 84f83f7..e44b742 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -7,13 +7,11 @@ def kmeans(query, no_clusters, no_init=20):
     """
 
     """
-    full_query = '''
-      SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,
-             array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,
-             array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys
-        FROM ({query}) As a
-       WHERE the_geom IS NOT NULL
-        '''.format(query=query)
+    full_query = ("SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,"
+                  "array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,"
+                  "array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys "
+                  "FROM ({query}) As a "
+                  "WHERE the_geom IS NOT NULL").format(query=query)
     try:
         data = plpy.execute(full_query)
     except plpy.SPIError, err:

From 3e0dba35221cdbc5a66f4e9064ba73dba81c8775 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 18 Oct 2016 21:13:34 -0600
Subject: [PATCH 16/40] update comments

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 84f83f7..21d76ea 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -5,7 +5,8 @@ import numpy as np
 
 def kmeans(query, no_clusters, no_init=20):
     """
-
+        find centers based on clusteres of latitude/longitude pairs
+        query: SQL query that has a WGS84 geometry (the_geom)
     """
     full_query = '''
       SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,
@@ -17,8 +18,9 @@ def kmeans(query, no_clusters, no_init=20):
     try:
         data = plpy.execute(full_query)
     except plpy.SPIError, err:
-        plpy.error("KMeans cluster failed: %s" % err)
+        plpy.error("k-means (spatial) cluster analysis failed: %s" % err)
 
+    # Unpack query response
     xs = data[0]['xs']
     ys = data[0]['ys']
     ids = data[0]['ids']
@@ -55,9 +57,9 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
     try:
         db_resp = plpy.execute(full_query)
     except plpy.SPIError, err:
-        plpy.error('k-means cluster analysis failed: %s' % err)
+        plpy.error("k-means (non-spatial) cluster analysis failed: %s" % err)
 
-    # fill array with values for kmeans clustering
+    # fill array with values for k-means clustering
     if standarize:
         cluster_columns = _scale_data(
           _extract_columns(db_resp, id_col=out_id_colname))

From 4389c9538d9c65c459dfaf4430683669362c4938 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 21 Oct 2016 10:13:21 -0600
Subject: [PATCH 17/40] small updates for readability

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 0df9ce0..df024a1 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -74,15 +74,15 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
                db_resp[0][out_id_colname])
 
 
-def _extract_columns(db_resp, id_col):
+def _extract_columns(db_resp, id_col_name):
     """
         Extract the features from the query and pack them into a NumPy array
         db_resp (plpy data object): result of the kmeans request
-        id_col (string): name of column which has the row id (not a feature of
-                         the analysis)
+        id_col_name (string): name of column which has the row id (not a
+                              feature of the analysis)
     """
     return np.array([db_resp[0][c] for c in db_resp.colnames()
-                     if c != id_col],
+                     if c != id_col_name],
                     dtype=float).T
 
 # -- Preprocessing steps
@@ -91,7 +91,8 @@ def _extract_columns(db_resp, id_col):
 def _scale_data(features):
     """
         Scale all input columns to center on 0 with a standard devation of 1
-        features (numpy array): an array of dimension (n_features, n_samples)
+
+        features (numpy matrix): features of dimension (n_features, n_samples)
     """
     from sklearn.preprocessing import StandardScaler
     return StandardScaler().fit_transform(features)

From a188b2e10415b91670f46401610d62f4ffa35316 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 21 Oct 2016 15:51:54 -0600
Subject: [PATCH 18/40] adds missing arguments

---
 src/pg/sql/11_kmeans.sql                          | 14 ++++++++++----
 src/py/crankshaft/crankshaft/clustering/kmeans.py |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 2db57e0..6a9d1a9 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -11,12 +11,18 @@ $$ LANGUAGE plpythonu;
 -- Non-spatial k-means clustering
 -- query: sql query to retrieve all the needed data
 
-CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(query TEXT, colnames TEXT[], num_clusters INTEGER, id_col TEXT DEFAULT 'cartodb_id')
+CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
+  query TEXT,
+  colnames TEXT[],
+  num_clusters INTEGER,
+  id_col TEXT DEFAULT 'cartodb_id',
+  standarize BOOLEAN DEFAULT true
+)
 RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint) AS $$
 
-from crankshaft.clustering import kmeans_nonspatial
-return kmeans_nonspatial(query, colnames, num_clusters, id_col)
-
+    from crankshaft.clustering import kmeans_nonspatial
+    return kmeans_nonspatial(query, colnames, num_clusters,
+                             id_col, standarize)
 $$ LANGUAGE plpythonu;
 
 
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index df024a1..6e972e5 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -62,7 +62,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
         cluster_columns = _scale_data(
           _extract_columns(db_resp, id_col=out_id_colname))
     else:
-        cluster_columns = _extract_columns(db_resp)
+        cluster_columns = _extract_columns(db_resp, id_col=out_id_colname)
 
     # TODO: decide on optimal parameters for most cases
     #       Are there ways of deciding parameters based on inputs?

From 64c4b6611c866853c7fb079d3465c7ee10e84f3c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 10 Nov 2016 16:56:04 +0000
Subject: [PATCH 19/40] changes cluster centers to json

---
 src/pg/sql/11_kmeans.sql                          | 6 +++---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 6a9d1a9..175ab6b 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -15,14 +15,14 @@ CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
   query TEXT,
   colnames TEXT[],
   num_clusters INTEGER,
-  id_col TEXT DEFAULT 'cartodb_id',
+  id_colname TEXT DEFAULT 'cartodb_id',
   standarize BOOLEAN DEFAULT true
 )
-RETURNS TABLE(cluster_label text, cluster_center text, rowid bigint) AS $$
+RETURNS TABLE(cluster_label text, cluster_center json, rowid bigint) AS $$
 
     from crankshaft.clustering import kmeans_nonspatial
     return kmeans_nonspatial(query, colnames, num_clusters,
-                             id_col, standarize)
+                             id_colname, standarize)
 $$ LANGUAGE plpythonu;
 
 
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 6e972e5..5bd7830 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -40,6 +40,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
         num_clusters (int): number of clusters (greater than zero)
         id_col (string): name of the input id_column
     """
+    import json
     out_id_colname = 'rowids'
     # TODO: need a random seed?
 
@@ -60,7 +61,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
     # fill array with values for k-means clustering
     if standarize:
         cluster_columns = _scale_data(
-          _extract_columns(db_resp, id_col=out_id_colname))
+          _extract_columns(db_resp, out_id_colname))
     else:
         cluster_columns = _extract_columns(db_resp, id_col=out_id_colname)
 
@@ -69,8 +70,9 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
     kmeans = KMeans(n_clusters=num_clusters,
                     random_state=0).fit(cluster_columns)
 
+    centers = [json.dumps(dict(zip(colnames, c))) for c in kmeans.cluster_centers_[kmeans.labels_]]
     return zip(kmeans.labels_,
-               map(str, kmeans.cluster_centers_[kmeans.labels_]),
+               centers,
                db_resp[0][out_id_colname])
 
 

From b6dae5e3801e7f33360c16172b41b4c62077614e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 00:15:23 +0100
Subject: [PATCH 20/40] adding silhouette

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 5bd7830..52139d1 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -41,6 +41,8 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
         id_col (string): name of the input id_column
     """
     import json
+    from sklearn import metrics
+
     out_id_colname = 'rowids'
     # TODO: need a random seed?
 
@@ -70,7 +72,13 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
     kmeans = KMeans(n_clusters=num_clusters,
                     random_state=0).fit(cluster_columns)
 
-    centers = [json.dumps(dict(zip(colnames, c))) for c in kmeans.cluster_centers_[kmeans.labels_]]
+    centers = [json.dumps(dict(zip(colnames, c)))
+               for c in kmeans.cluster_centers_[kmeans.labels_]]
+
+    silhouettes = metrics.silhouette_samples(cluster_columns,
+                                             labels,
+                                             metric='sqeuclidean')
+
     return zip(kmeans.labels_,
                centers,
                db_resp[0][out_id_colname])

From af536757febee26ae7a699d755209e35fc57c246 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Mon, 14 Nov 2016 23:29:38 +0000
Subject: [PATCH 21/40] adds silhouettes to output

---
 src/pg/sql/11_kmeans.sql                          | 2 +-
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 175ab6b..c9ae131 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -18,7 +18,7 @@ CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
   id_colname TEXT DEFAULT 'cartodb_id',
   standarize BOOLEAN DEFAULT true
 )
-RETURNS TABLE(cluster_label text, cluster_center json, rowid bigint) AS $$
+RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
 
     from crankshaft.clustering import kmeans_nonspatial
     return kmeans_nonspatial(query, colnames, num_clusters,
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 52139d1..d070bf9 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -76,11 +76,12 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
                for c in kmeans.cluster_centers_[kmeans.labels_]]
 
     silhouettes = metrics.silhouette_samples(cluster_columns,
-                                             labels,
+                                             kmeans.labels_,
                                              metric='sqeuclidean')
 
     return zip(kmeans.labels_,
                centers,
+               silhouettes,
                db_resp[0][out_id_colname])
 
 

From cbe8571546e2f89170ad7d294d4af7022112f3fe Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 10:10:07 +0100
Subject: [PATCH 22/40] fixes argument in not-standardize

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index d070bf9..383584e 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -65,7 +65,7 @@ def kmeans_nonspatial(query, colnames, num_clusters=5,
         cluster_columns = _scale_data(
           _extract_columns(db_resp, out_id_colname))
     else:
-        cluster_columns = _extract_columns(db_resp, id_col=out_id_colname)
+        cluster_columns = _extract_columns(db_resp, out_id_colname)
 
     # TODO: decide on optimal parameters for most cases
     #       Are there ways of deciding parameters based on inputs?

From 0867e69d1f21cae22c611612cf9fbe197c09c284 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 11:19:15 +0100
Subject: [PATCH 23/40] replace plpy method colnames

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 383584e..e4fc7e5 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -92,7 +92,9 @@ def _extract_columns(db_resp, id_col_name):
         id_col_name (string): name of column which has the row id (not a
                               feature of the analysis)
     """
-    return np.array([db_resp[0][c] for c in db_resp.colnames()
+    keys = [k for k in db_resp[0].keys()]
+
+    return np.array([db_resp[0][c] for c in keys
                      if c != id_col_name],
                     dtype=float).T
 

From 0d40080f6cdd1b5a47e70377b0317493d660081f Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 12:02:42 +0100
Subject: [PATCH 24/40] move back to colnames

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index e4fc7e5..383584e 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -92,9 +92,7 @@ def _extract_columns(db_resp, id_col_name):
         id_col_name (string): name of column which has the row id (not a
                               feature of the analysis)
     """
-    keys = [k for k in db_resp[0].keys()]
-
-    return np.array([db_resp[0][c] for c in keys
+    return np.array([db_resp[0][c] for c in db_resp.colnames()
                      if c != id_col_name],
                     dtype=float).T
 

From ded26dc46bac37f13c17461ec6280d0109d62be3 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 12:03:24 +0100
Subject: [PATCH 25/40] adding class for database response

---
 src/py/crankshaft/test/helper.py    |  1 +
 src/py/crankshaft/test/mock_plpy.py | 27 +++++++++++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/py/crankshaft/test/helper.py b/src/py/crankshaft/test/helper.py
index 7d28b94..b273354 100644
--- a/src/py/crankshaft/test/helper.py
+++ b/src/py/crankshaft/test/helper.py
@@ -2,6 +2,7 @@ import unittest
 
 from mock_plpy import MockPlPy
 plpy = MockPlPy()
+from mock_plpy import MockDBResponse
 
 import sys
 sys.modules['plpy'] = plpy
diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py
index a982ebe..05d0f21 100644
--- a/src/py/crankshaft/test/mock_plpy.py
+++ b/src/py/crankshaft/test/mock_plpy.py
@@ -1,12 +1,13 @@
 import re
 
+
 class MockCursor:
     def __init__(self, data):
         self.cursor_pos = 0
         self.data = data
 
     def fetch(self, batch_size):
-        batch = self.data[self.cursor_pos : self.cursor_pos + batch_size]
+        batch = self.data[self.cursor_pos:self.cursor_pos + batch_size]
         self.cursor_pos += batch_size
         return batch
 
@@ -45,8 +46,22 @@ class MockPlPy:
         data = self.execute(query)
         return MockCursor(data)
 
-    def execute(self, query): # TODO: additional arguments
-       for result in self.results:
-          if result[0].match(query):
-            return result[1]
-       return []
+    # TODO: additional arguments
+    def execute(self, query):
+        for result in self.results:
+            if result[0].match(query):
+                return result[1]
+        return []
+
+
+class MockDBResponse:
+    def __init__(self, data, colnames=None):
+        self.data = data
+        if colnames is None:
+            self.colnames = data[0].keys()
+        else:
+            self.colnames = colnames
+
+
+    def colnames(self):
+        return self.colnames

From 84d33d841f84e6ff8aafdd6c2dcf5b0727cc95c6 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 15 Nov 2016 12:03:54 +0100
Subject: [PATCH 26/40] tests for new class

---
 .../crankshaft/test/test_clustering_kmeans.py | 38 ++++++++++++++++++-
 .../crankshaft/test/test_clustering_moran.py  | 15 +++++---
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index e130da4..03cbd0a 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -7,9 +7,10 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
+from helper import plpy, fixture_file, MockDBResponse
 import crankshaft.clustering as cc
 import json
+from collections import OrderedDict
 
 
 class KMeansTest(unittest.TestCase):
@@ -38,3 +39,38 @@ class KMeansTest(unittest.TestCase):
         self.assertEqual(len(np.unique(labels)), 2)
         self.assertEqual(len(c1), 20)
         self.assertEqual(len(c2), 20)
+
+
+class KMeansNonspatialTest(unittest.TestCase):
+    """Testing class for k-means non-spatial"""
+
+    def setUp(self):
+        plpy._reset()
+
+        # self.cluster_data = json.loads(
+        #     open(fixture_file('kmeans-nonspatial.json')).read())
+
+        self.params = {"subquery": "SELECT * FROM TABLE",
+                       "n_clusters": 5}
+
+    def test_kmeans_nonspatial(self):
+        """
+            test for k-means non-spatial
+        """
+        data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
+                                 ("col2", [2, 4, 0, 2, 4, 0]),
+                                 ("rowids", [1, 2, 3, 4, 5, 6])])]
+
+        data_obj = MockDBResponse(data_raw, [k for k in data_raw[0]
+                                             if k != 'rowids'])
+        plpy._define_result('select', data_obj)
+        clusters = cc.kmeans_nonspatial('subquery', ['col1', 'col2'], 4)
+
+        cl1 = clusters[0][1]
+        cl2 = clusters[3][1]
+
+        for idx, val in enumerate(clusters):
+            if idx < 3:
+                self.assertEqual(val[1], cl1)
+            else:
+                self.assertEqual(val[1], cl2)
diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py
index cb54902..83256ad 100644
--- a/src/py/crankshaft/test/test_clustering_moran.py
+++ b/src/py/crankshaft/test/test_clustering_moran.py
@@ -7,13 +7,13 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
+from helper import plpy, fixture_file, MockDBResponse
 
 import crankshaft.clustering as cc
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
-
+from collections import OrderedDict
 
 class MoranTest(unittest.TestCase):
     """Testing class for Moran's I functions"""
@@ -58,11 +58,14 @@ class MoranTest(unittest.TestCase):
 
     def test_moran_local(self):
         """Test Moran's I local"""
-        data = [{'id': d['id'],
-                 'attr1': d['value'],
-                 'neighbors': d['neighbors']} for d in self.neighbors_data]
+        data = [OrderedDict([('id', d['id']),
+                             ('attr1', d['value']),
+                             ('neighbors', d['neighbors'])])
+                for d in self.neighbors_data]
 
-        plpy._define_result('select', data)
+        db_resp = MockDBResponse(data)
+
+        plpy._define_result('select', db_resp)
         random_seeds.set_random_seeds(1234)
         result = cc.moran_local('subquery', 'value',
                                 'knn', 5, 99, 'the_geom', 'cartodb_id')

From 7eee4faac1d5fb966ed74a8d4afdbb05d7c3f20c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 18 Nov 2016 17:22:02 +0000
Subject: [PATCH 27/40] rename to match numbering elsewhere

---
 src/pg/test/sql/{05_kmeans_test.sql => 11_kmeans_test.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/pg/test/sql/{05_kmeans_test.sql => 11_kmeans_test.sql} (100%)

diff --git a/src/pg/test/sql/05_kmeans_test.sql b/src/pg/test/sql/11_kmeans_test.sql
similarity index 100%
rename from src/pg/test/sql/05_kmeans_test.sql
rename to src/pg/test/sql/11_kmeans_test.sql

From 83f1900512715705dc44871b70215f048cdfbc00 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 18 Nov 2016 17:24:18 +0000
Subject: [PATCH 28/40] creates class-based approach to analysis methods

---
 src/pg/sql/11_kmeans.sql                      |  10 +-
 .../crankshaft/clustering/kmeans.py           | 180 +++++++++++-------
 .../crankshaft/test/test_clustering_kmeans.py |  45 +++--
 3 files changed, 143 insertions(+), 92 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index c9ae131..1dc6d00 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -3,8 +3,9 @@
 CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer, no_init integer default 20)
 RETURNS table (cartodb_id integer, cluster_no integer) as $$
 
-    from crankshaft.clustering import kmeans
-    return kmeans(query, no_clusters, no_init)
+    from crankshaft.clustering import Kmeans
+    kmeans = Kmeans()
+    return kmeans.spatial(query, no_clusters, no_init)
 
 $$ LANGUAGE plpythonu;
 
@@ -20,8 +21,9 @@ CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
 )
 RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
 
-    from crankshaft.clustering import kmeans_nonspatial
-    return kmeans_nonspatial(query, colnames, num_clusters,
+    from crankshaft.clustering import Kmeans
+    kmeans = Kmeans()
+    return kmeans.nonspatial(query, colnames, num_clusters,
                              id_colname, standarize)
 $$ LANGUAGE plpythonu;
 
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 383584e..48b9bd3 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -3,101 +3,135 @@ import plpy
 import numpy as np
 
 
-def kmeans(query, no_clusters, no_init=20):
-    """
-        find centers based on clusteres of latitude/longitude pairs
-        query: SQL query that has a WGS84 geometry (the_geom)
-    """
-    full_query = ("SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,"
-                  "array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,"
-                  "array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys "
-                  "FROM ({query}) As a "
-                  "WHERE the_geom IS NOT NULL").format(query=query)
-    try:
-        data = plpy.execute(full_query)
-    except plpy.SPIError, err:
-        plpy.error("k-means (spatial) cluster analysis failed: %s" % err)
+class QueryRunner:
+    def get_moran(self, query):
+        """fetch data for moran's i analyses"""
+        try:
+            result = plpy.execute(query)
+            # if there are no neighbors, exit
+            if len(result) == 0:
+                return pu.empty_zipped_array(2)
+        except plpy.SPIError, e:
+            plpy.error('Analysis failed: %s' % e)
+            return pu.empty_zipped_array(2)
 
-    # Unpack query response
-    xs = data[0]['xs']
-    ys = data[0]['ys']
-    ids = data[0]['ids']
+    def get_columns(self, query, standarize):
+        """fetch data for non-spatial kmeans"""
+        try:
+            db_resp = plpy.execute(query)
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % err)
 
-    km = KMeans(n_clusters=no_clusters, n_init=no_init)
-    labels = km.fit_predict(zip(xs, ys))
-    return zip(ids, labels)
+        return db_resp
+
+    def get_result(self, query):
+        """fetch data for spatial kmeans"""
+        try:
+            data = plpy.execute(query)
+        except plpy.SPIError, err:
+            plpy.error("Analysis failed: %s" % err)
+        return data
 
 
-def kmeans_nonspatial(query, colnames, num_clusters=5,
-                      id_col='cartodb_id', standarize=True):
-    """
-        query (string): A SQL query to retrieve the data required to do the
-                        k-means clustering analysis, like so:
-                        SELECT * FROM iris_flower_data
-        colnames (list): a list of the column names which contain the data of
-                         interest, like so: ["sepal_width", "petal_width",
-                                             "sepal_length", "petal_length"]
-        num_clusters (int): number of clusters (greater than zero)
-        id_col (string): name of the input id_column
-    """
-    import json
-    from sklearn import metrics
+class Kmeans:
+    def __init__(self, query_runner=None):
+        if query_runner is None:
+            self.query_runner = QueryRunner()
+        else:
+            self.query_runner = query_runner
 
-    out_id_colname = 'rowids'
-    # TODO: need a random seed?
+    def spatial(self, query, no_clusters, no_init=20):
+        """
+            find centers based on clusters of latitude/longitude pairs
+            query: SQL query that has a WGS84 geometry (the_geom)
+        """
+        full_query = ("SELECT "
+                      "array_agg(cartodb_id ORDER BY cartodb_id) as ids,"
+                      "array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,"
+                      "array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys "
+                      "FROM ({query}) As a "
+                      "WHERE the_geom IS NOT NULL").format(query=query)
 
-    full_query = '''
-        SELECT {cols}, array_agg({id_col}) As {out_id_colname}
-        FROM ({query}) As a
-    '''.format(query=query,
-               id_col=id_col,
-               out_id_colname=out_id_colname,
-               cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
-                               for idx, val in enumerate(colnames)]))
+        data = self.query_runner.get_result(full_query)
 
-    try:
-        db_resp = plpy.execute(full_query)
-    except plpy.SPIError, err:
-        plpy.error("k-means (non-spatial) cluster analysis failed: %s" % err)
+        # Unpack query response
+        xs = data[0]['xs']
+        ys = data[0]['ys']
+        ids = data[0]['ids']
 
-    # fill array with values for k-means clustering
-    if standarize:
-        cluster_columns = _scale_data(
-          _extract_columns(db_resp, out_id_colname))
-    else:
-        cluster_columns = _extract_columns(db_resp, out_id_colname)
+        km = KMeans(n_clusters=no_clusters, n_init=no_init)
+        labels = km.fit_predict(zip(xs, ys))
+        return zip(ids, labels)
 
-    # TODO: decide on optimal parameters for most cases
-    #       Are there ways of deciding parameters based on inputs?
-    kmeans = KMeans(n_clusters=num_clusters,
-                    random_state=0).fit(cluster_columns)
+    def nonspatial(self, query, colnames, num_clusters=5,
+                   id_col='cartodb_id', standarize=True):
+        """
+            query (string): A SQL query to retrieve the data required to do the
+                            k-means clustering analysis, like so:
+                            SELECT * FROM iris_flower_data
+            colnames (list): a list of the column names which contain the data
+                             of interest, like so: ["sepal_width",
+                                                    "petal_width",
+                                                    "sepal_length",
+                                                    "petal_length"]
+            num_clusters (int): number of clusters (greater than zero)
+            id_col (string): name of the input id_column
+        """
+        import json
+        from sklearn import metrics
 
-    centers = [json.dumps(dict(zip(colnames, c)))
-               for c in kmeans.cluster_centers_[kmeans.labels_]]
+        out_id_colname = 'rowids'
+        # TODO: need a random seed?
 
-    silhouettes = metrics.silhouette_samples(cluster_columns,
-                                             kmeans.labels_,
-                                             metric='sqeuclidean')
+        full_query = '''
+            SELECT {cols}, array_agg({id_col}) As {out_id_colname}
+            FROM ({query}) As a
+        '''.format(query=query,
+                   id_col=id_col,
+                   out_id_colname=out_id_colname,
+                   cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
+                                   for idx, val in enumerate(colnames)]))
 
-    return zip(kmeans.labels_,
-               centers,
-               silhouettes,
-               db_resp[0][out_id_colname])
+        db_resp = self.query_runner.get_columns(full_query, standarize)
+
+        # fill array with values for k-means clustering
+        if standarize:
+            cluster_columns = _scale_data(
+              _extract_columns(db_resp, colnames))
+        else:
+            cluster_columns = _extract_columns(db_resp, colnames)
+
+        print str(cluster_columns)
+        # TODO: decide on optimal parameters for most cases
+        #       Are there ways of deciding parameters based on inputs?
+        kmeans = KMeans(n_clusters=num_clusters,
+                        random_state=0).fit(cluster_columns)
+
+        centers = [json.dumps(dict(zip(colnames, c)))
+                   for c in kmeans.cluster_centers_[kmeans.labels_]]
+
+        silhouettes = metrics.silhouette_samples(cluster_columns,
+                                                 kmeans.labels_,
+                                                 metric='sqeuclidean')
+
+        return zip(kmeans.labels_,
+                   centers,
+                   silhouettes,
+                   db_resp[0][out_id_colname])
 
 
-def _extract_columns(db_resp, id_col_name):
+# -- Preprocessing steps
+
+def _extract_columns(db_resp, colnames):
     """
         Extract the features from the query and pack them into a NumPy array
         db_resp (plpy data object): result of the kmeans request
         id_col_name (string): name of column which has the row id (not a
                               feature of the analysis)
     """
-    return np.array([db_resp[0][c] for c in db_resp.colnames()
-                     if c != id_col_name],
+    return np.array([db_resp[0][c] for c in colnames],
                     dtype=float).T
 
-# -- Preprocessing steps
-
 
 def _scale_data(features):
     """
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 03cbd0a..8e5c9b4 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -7,17 +7,31 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file, MockDBResponse
+from helper import plpy, fixture_file
+from crankshaft.clustering import Kmeans
+from crankshaft.clustering import QueryRunner
 import crankshaft.clustering as cc
+
+from crankshaft import random_seeds
 import json
 from collections import OrderedDict
 
 
+class FakeQueryRunner(QueryRunner):
+    def __init__(self, mocked_result):
+        self.mocked_result = mocked_result
+
+    def get_result(self, query):
+        return self.mocked_result
+
+    def get_columns(self, query, standarize):
+        return self.mocked_result
+
+
 class KMeansTest(unittest.TestCase):
     """Testing class for k-means spatial"""
 
     def setUp(self):
-        plpy._reset()
         self.cluster_data = json.loads(
           open(fixture_file('kmeans.json')).read())
         self.params = {"subquery": "select * from table",
@@ -30,8 +44,9 @@ class KMeansTest(unittest.TestCase):
                  'ys': d['ys'],
                  'ids': d['ids']} for d in self.cluster_data]
 
-        plpy._define_result('select', data)
-        clusters = cc.kmeans('subquery', 2)
+        random_seeds.set_random_seeds(1234)
+        kmeans = Kmeans(FakeQueryRunner(data))
+        clusters = kmeans.spatial('subquery', 2)
         labels = [a[1] for a in clusters]
         c1 = [a for a in clusters if a[1] == 0]
         c2 = [a for a in clusters if a[1] == 1]
@@ -47,9 +62,6 @@ class KMeansNonspatialTest(unittest.TestCase):
     def setUp(self):
         plpy._reset()
 
-        # self.cluster_data = json.loads(
-        #     open(fixture_file('kmeans-nonspatial.json')).read())
-
         self.params = {"subquery": "SELECT * FROM TABLE",
                        "n_clusters": 5}
 
@@ -57,20 +69,23 @@ class KMeansNonspatialTest(unittest.TestCase):
         """
             test for k-means non-spatial
         """
+        # data from:
+        # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans
         data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
                                  ("col2", [2, 4, 0, 2, 4, 0]),
                                  ("rowids", [1, 2, 3, 4, 5, 6])])]
 
-        data_obj = MockDBResponse(data_raw, [k for k in data_raw[0]
-                                             if k != 'rowids'])
-        plpy._define_result('select', data_obj)
-        clusters = cc.kmeans_nonspatial('subquery', ['col1', 'col2'], 4)
+        random_seeds.set_random_seeds(1234)
+        kmeans = Kmeans(FakeQueryRunner(data_raw))
+        print 'asfasdfasd'
+        clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2)
+        print str([c[0] for c in clusters])
 
-        cl1 = clusters[0][1]
-        cl2 = clusters[3][1]
+        cl1 = clusters[0][0]
+        cl2 = clusters[3][0]
 
         for idx, val in enumerate(clusters):
             if idx < 3:
-                self.assertEqual(val[1], cl1)
+                self.assertEqual(val[0], cl1)
             else:
-                self.assertEqual(val[1], cl2)
+                self.assertEqual(val[0], cl2)

From a9add4b49c49c5483f5aed13598ac076b376d8c1 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 18 Nov 2016 17:40:57 +0000
Subject: [PATCH 29/40] rename results file

---
 src/pg/test/expected/{05_kmeans_test.out => 11_kmeans_test.out} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/pg/test/expected/{05_kmeans_test.out => 11_kmeans_test.out} (100%)

diff --git a/src/pg/test/expected/05_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
similarity index 100%
rename from src/pg/test/expected/05_kmeans_test.out
rename to src/pg/test/expected/11_kmeans_test.out

From a8bd122762c321a14263a528d6265089dbdaba93 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 18 Nov 2016 17:46:29 +0000
Subject: [PATCH 30/40] remove mock plpy dependencies

---
 src/py/crankshaft/test/helper.py              |  1 -
 src/py/crankshaft/test/mock_plpy.py           | 13 -----
 .../crankshaft/test/test_clustering_kmeans.py |  4 +-
 .../crankshaft/test/test_clustering_moran.py  | 58 +++++++++++--------
 4 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/src/py/crankshaft/test/helper.py b/src/py/crankshaft/test/helper.py
index b273354..7d28b94 100644
--- a/src/py/crankshaft/test/helper.py
+++ b/src/py/crankshaft/test/helper.py
@@ -2,7 +2,6 @@ import unittest
 
 from mock_plpy import MockPlPy
 plpy = MockPlPy()
-from mock_plpy import MockDBResponse
 
 import sys
 sys.modules['plpy'] = plpy
diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py
index 05d0f21..e8a279d 100644
--- a/src/py/crankshaft/test/mock_plpy.py
+++ b/src/py/crankshaft/test/mock_plpy.py
@@ -52,16 +52,3 @@ class MockPlPy:
             if result[0].match(query):
                 return result[1]
         return []
-
-
-class MockDBResponse:
-    def __init__(self, data, colnames=None):
-        self.data = data
-        if colnames is None:
-            self.colnames = data[0].keys()
-        else:
-            self.colnames = colnames
-
-
-    def colnames(self):
-        return self.colnames
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 8e5c9b4..830ee9d 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -7,7 +7,7 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
+from helper import fixture_file
 from crankshaft.clustering import Kmeans
 from crankshaft.clustering import QueryRunner
 import crankshaft.clustering as cc
@@ -60,8 +60,6 @@ class KMeansNonspatialTest(unittest.TestCase):
     """Testing class for k-means non-spatial"""
 
     def setUp(self):
-        plpy._reset()
-
         self.params = {"subquery": "SELECT * FROM TABLE",
                        "n_clusters": 5}
 
diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py
index 83256ad..0a320fb 100644
--- a/src/py/crankshaft/test/test_clustering_moran.py
+++ b/src/py/crankshaft/test/test_clustering_moran.py
@@ -7,19 +7,30 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file, MockDBResponse
-
-import crankshaft.clustering as cc
+from helper import fixture_file
+from crankshaft.clustering import Moran
+from crankshaft.clustering import QueryRunner
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
 from collections import OrderedDict
 
+
+class FakeQueryRunner(QueryRunner):
+    def __init__(self, mocked_result):
+        self.mocked_result = mocked_result
+
+    def get_result(self, query):
+        return self.mocked_result
+
+    def get_columns(self, query):
+        return self.mocked_result
+
+
 class MoranTest(unittest.TestCase):
     """Testing class for Moran's I functions"""
 
     def setUp(self):
-        plpy._reset()
         self.params = {"id_col": "cartodb_id",
                        "attr1": "andy",
                        "attr2": "jay_z",
@@ -39,36 +50,36 @@ class MoranTest(unittest.TestCase):
 
     def test_map_quads(self):
         """Test map_quads"""
-        self.assertEqual(cc.map_quads(1), 'HH')
-        self.assertEqual(cc.map_quads(2), 'LH')
-        self.assertEqual(cc.map_quads(3), 'LL')
-        self.assertEqual(cc.map_quads(4), 'HL')
-        self.assertEqual(cc.map_quads(33), None)
-        self.assertEqual(cc.map_quads('andy'), None)
+        from crankshaft.clustering import map_quads
+        self.assertEqual(map_quads(1), 'HH')
+        self.assertEqual(map_quads(2), 'LH')
+        self.assertEqual(map_quads(3), 'LL')
+        self.assertEqual(map_quads(4), 'HL')
+        self.assertEqual(map_quads(33), None)
+        self.assertEqual(map_quads('andy'), None)
 
     def test_quad_position(self):
         """Test lisa_sig_vals"""
+        from crankshaft.clustering import quad_position
 
         quads = np.array([1, 2, 3, 4], np.int)
 
         ans = np.array(['HH', 'LH', 'LL', 'HL'])
-        test_ans = cc.quad_position(quads)
+        test_ans = quad_position(quads)
 
         self.assertTrue((test_ans == ans).all())
 
-    def test_moran_local(self):
+    def test_local_stat(self):
         """Test Moran's I local"""
         data = [OrderedDict([('id', d['id']),
                              ('attr1', d['value']),
                              ('neighbors', d['neighbors'])])
                 for d in self.neighbors_data]
 
-        db_resp = MockDBResponse(data)
-
-        plpy._define_result('select', db_resp)
+        moran = Moran(FakeQueryRunner(data))
         random_seeds.set_random_seeds(1234)
-        result = cc.moran_local('subquery', 'value',
-                                'knn', 5, 99, 'the_geom', 'cartodb_id')
+        result = moran.local_stat('subquery', 'value',
+                                  'knn', 5, 99, 'the_geom', 'cartodb_id')
         result = [(row[0], row[1]) for row in result]
         zipped_values = zip(result, self.moran_data)
 
@@ -83,10 +94,10 @@ class MoranTest(unittest.TestCase):
                  'attr2': 1,
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
 
-        plpy._define_result('select', data)
         random_seeds.set_random_seeds(1234)
-        result = cc.moran_local_rate('subquery', 'numerator', 'denominator',
-                                     'knn', 5, 99, 'the_geom', 'cartodb_id')
+        moran = Moran(FakeQueryRunner(data))
+        result = moran.local_rate_stat('subquery', 'numerator', 'denominator',
+                                       'knn', 5, 99, 'the_geom', 'cartodb_id')
         result = [(row[0], row[1]) for row in result]
 
         zipped_values = zip(result, self.moran_data)
@@ -99,10 +110,11 @@ class MoranTest(unittest.TestCase):
         data = [{'id': d['id'],
                  'attr1': d['value'],
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
-        plpy._define_result('select', data)
         random_seeds.set_random_seeds(1235)
-        result = cc.moran('table', 'value',
-                          'knn', 5, 99, 'the_geom', 'cartodb_id')
+        moran = Moran(FakeQueryRunner(data))
+        result = moran.global_stat('table', 'value',
+                                   'knn', 5, 99, 'the_geom',
+                                   'cartodb_id')
 
         result_moran = result[0][0]
         expected_moran = np.array([row[0] for row in self.moran_data]).mean()

From 2738c1f29cd559fb0db174fad74fcde46f463844 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Fri, 18 Nov 2016 17:46:55 +0000
Subject: [PATCH 31/40] move to class-based module

---
 src/pg/sql/10_moran.sql                       |  23 +-
 .../crankshaft/crankshaft/clustering/moran.py | 295 ++++++++----------
 2 files changed, 151 insertions(+), 167 deletions(-)

diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql
index 070392d..bd3f96d 100644
--- a/src/pg/sql/10_moran.sql
+++ b/src/pg/sql/10_moran.sql
@@ -10,9 +10,11 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE (moran NUMERIC, significance NUMERIC)
 AS $$
-  from crankshaft.clustering import moran
+  from crankshaft.clustering import Moran
   # TODO: use named parameters or a dictionary
-  return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
+  moran = Moran()
+  return moran.global_stat(subquery, column_name, w_type,
+                           num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 -- Moran's I Local (internal function)
@@ -27,9 +29,11 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT)
 RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
 AS $$
-  from crankshaft.clustering import moran_local
+  from crankshaft.clustering import Moran
+  moran = Moran()
   # TODO: use named parameters or a dictionary
-  return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
+  return moran.local_stat(subquery, column_name, w_type,
+                          num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 -- Moran's I Local (public-facing function)
@@ -120,9 +124,11 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE (moran FLOAT, significance FLOAT)
 AS $$
-  from crankshaft.clustering import moran_local
+  from crankshaft.clustering import Moran
+  moran = Moran()
   # TODO: use named parameters or a dictionary
-  return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
+  return moran.global_rate_stat(subquery, numerator, denominator, w_type,
+                                num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 
@@ -140,9 +146,10 @@ CREATE OR REPLACE FUNCTION
 RETURNS
 TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
 AS $$
-  from crankshaft.clustering import moran_local_rate
+  from crankshaft.clustering import Moran
+  moran = Moran()
   # TODO: use named parameters or a dictionary
-  return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
+  return moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 -- Moran's I Local Rate (public-facing function)
diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py
index 4e7086e..ee82932 100644
--- a/src/py/crankshaft/crankshaft/clustering/moran.py
+++ b/src/py/crankshaft/crankshaft/clustering/moran.py
@@ -15,204 +15,181 @@ import crankshaft.pysal_utils as pu
 # High level interface ---------------------------------------
 
 
-def moran(subquery, attr_name,
-          w_type, num_ngbrs, permutations, geom_col, id_col):
-    """
-    Moran's I (global)
-    Implementation building neighbors with a PostGIS database and Moran's I
-     core clusters with PySAL.
-    Andy Eschbacher
-    """
-    qvals = OrderedDict([("id_col", id_col),
-                         ("attr1", attr_name),
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
-
-    query = pu.construct_neighbor_query(w_type, qvals)
-
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(2)
-    except plpy.SPIError, e:
-        plpy.error('Analysis failed: %s' % e)
-        return pu.empty_zipped_array(2)
-
-    # collect attributes
-    attr_vals = pu.get_attributes(result)
-
-    # calculate weights
-    weight = pu.get_weight(result, w_type, num_ngbrs)
-
-    # calculate moran global
-    moran_global = ps.esda.moran.Moran(attr_vals, weight,
-                                       permutations=permutations)
-
-    return zip([moran_global.I], [moran_global.EI])
+class QueryRunner:
+    def get_result(self, query):
+        try:
+            data = plpy.execute(query)
+        except plpy.SPIError, err:
+            plpy.error("k-means (spatial) cluster analysis failed: %s" % err)
+        return data
 
 
-def moran_local(subquery, attr,
-                w_type, num_ngbrs, permutations, geom_col, id_col):
-    """
-    Moran's I implementation for PL/Python
-    Andy Eschbacher
-    """
+class Moran:
+    def __init__(self, query_runner=None):
+        if query_runner is None:
+            self.query_runner = QueryRunner()
+        else:
+            self.query_runner = query_runner
 
-    # geometries with attributes that are null are ignored
-    # resulting in a collection of not as near neighbors
+    def global_stat(self, subquery, attr_name,
+                    w_type, num_ngbrs, permutations, geom_col, id_col):
+        """
+        Moran's I (global)
+        Implementation building neighbors with a PostGIS database and Moran's I
+         core clusters with PySAL.
+        Andy Eschbacher
+        """
+        qvals = OrderedDict([("id_col", id_col),
+                             ("attr1", attr_name),
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
-    qvals = OrderedDict([("id_col", id_col),
-                         ("attr1", attr),
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
+        query = pu.construct_neighbor_query(w_type, qvals)
 
-    query = pu.construct_neighbor_query(w_type, qvals)
+        result = self.query_runner.get_result(query)
 
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(5)
-    except plpy.SPIError, e:
-        plpy.error('Analysis failed: %s' % e)
-        return pu.empty_zipped_array(5)
+        # collect attributes
+        attr_vals = pu.get_attributes(result)
 
-    attr_vals = pu.get_attributes(result)
-    weight = pu.get_weight(result, w_type, num_ngbrs)
+        # calculate weights
+        weight = pu.get_weight(result, w_type, num_ngbrs)
 
-    # calculate LISA values
-    lisa = ps.esda.moran.Moran_Local(attr_vals, weight,
-                                     permutations=permutations)
+        # calculate moran global
+        moran_global = ps.esda.moran.Moran(attr_vals, weight,
+                                           permutations=permutations)
 
-    # find quadrants for each geometry
-    quads = quad_position(lisa.q)
+        return zip([moran_global.I], [moran_global.EI])
 
-    return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
+    def local_stat(self, subquery, attr,
+                   w_type, num_ngbrs, permutations, geom_col, id_col):
+        """
+        Moran's I implementation for PL/Python
+        Andy Eschbacher
+        """
 
+        # geometries with attributes that are null are ignored
+        # resulting in a collection of not as near neighbors
 
-def moran_rate(subquery, numerator, denominator,
-               w_type, num_ngbrs, permutations, geom_col, id_col):
-    """
-    Moran's I Rate (global)
-    Andy Eschbacher
-    """
-    qvals = OrderedDict([("id_col", id_col),
-                         ("attr1", numerator),
-                         ("attr2", denominator)
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
+        qvals = OrderedDict([("id_col", id_col),
+                             ("attr1", attr),
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
-    query = pu.construct_neighbor_query(w_type, qvals)
+        query = pu.construct_neighbor_query(w_type, qvals)
 
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(2)
-    except plpy.SPIError, e:
-        plpy.error('Analysis failed: %s' % e)
-        return pu.empty_zipped_array(2)
+        result = self.query_runner.get_result(query)
 
-    # collect attributes
-    numer = pu.get_attributes(result, 1)
-    denom = pu.get_attributes(result, 2)
+        attr_vals = pu.get_attributes(result)
+        weight = pu.get_weight(result, w_type, num_ngbrs)
 
-    weight = pu.get_weight(result, w_type, num_ngbrs)
-
-    # calculate moran global rate
-    lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight,
+        # calculate LISA values
+        lisa = ps.esda.moran.Moran_Local(attr_vals, weight,
                                          permutations=permutations)
 
-    return zip([lisa_rate.I], [lisa_rate.EI])
+        # find quadrants for each geometry
+        quads = quad_position(lisa.q)
 
+        return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
 
-def moran_local_rate(subquery, numerator, denominator,
-                     w_type, num_ngbrs, permutations, geom_col, id_col):
-    """
-        Moran's I Local Rate
+    def global_rate_stat(self, subquery, numerator, denominator,
+                         w_type, num_ngbrs, permutations, geom_col, id_col):
+        """
+        Moran's I Rate (global)
         Andy Eschbacher
-    """
-    # geometries with values that are null are ignored
-    # resulting in a collection of not as near neighbors
+        """
+        qvals = OrderedDict([("id_col", id_col),
+                             ("attr1", numerator),
+                             ("attr2", denominator)
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
-    qvals = OrderedDict([("id_col", id_col),
-                         ("numerator", numerator),
-                         ("denominator", denominator),
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
+        query = pu.construct_neighbor_query(w_type, qvals)
 
-    query = pu.construct_neighbor_query(w_type, qvals)
+        result = self.query_runner.get_result(query)
 
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(5)
-    except plpy.SPIError, e:
-        plpy.error('Analysis failed: %s' % e)
-        return pu.empty_zipped_array(5)
+        # collect attributes
+        numer = pu.get_attributes(result, 1)
+        denom = pu.get_attributes(result, 2)
 
-    # collect attributes
-    numer = pu.get_attributes(result, 1)
-    denom = pu.get_attributes(result, 2)
+        weight = pu.get_weight(result, w_type, num_ngbrs)
 
-    weight = pu.get_weight(result, w_type, num_ngbrs)
+        # calculate moran global rate
+        lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight,
+                                             permutations=permutations)
 
-    # calculate LISA values
-    lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight,
-                                          permutations=permutations)
+        return zip([lisa_rate.I], [lisa_rate.EI])
 
-    # find quadrants for each geometry
-    quads = quad_position(lisa.q)
+    def local_rate_stat(self, subquery, numerator, denominator,
+                        w_type, num_ngbrs, permutations, geom_col, id_col):
+        """
+            Moran's I Local Rate
+            Andy Eschbacher
+        """
+        # geometries with values that are null are ignored
+        # resulting in a collection of not as near neighbors
 
-    return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
+        qvals = OrderedDict([("id_col", id_col),
+                             ("numerator", numerator),
+                             ("denominator", denominator),
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
+        query = pu.construct_neighbor_query(w_type, qvals)
 
-def moran_local_bv(subquery, attr1, attr2,
-                   permutations, geom_col, id_col, w_type, num_ngbrs):
-    """
-        Moran's I (local) Bivariate (untested)
-    """
+        result = self.query_runner.get_result(query)
 
-    qvals = OrderedDict([("id_col", id_col),
-                         ("attr1", attr1),
-                         ("attr2", attr2),
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
+        # collect attributes
+        numer = pu.get_attributes(result, 1)
+        denom = pu.get_attributes(result, 2)
 
-    query = pu.construct_neighbor_query(w_type, qvals)
+        weight = pu.get_weight(result, w_type, num_ngbrs)
 
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(4)
-    except plpy.SPIError:
-        plpy.error("Error: areas of interest query failed, "
-                   "check input parameters")
-        return pu.empty_zipped_array(4)
+        # calculate LISA values
+        lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight,
+                                              permutations=permutations)
 
-    # collect attributes
-    attr1_vals = pu.get_attributes(result, 1)
-    attr2_vals = pu.get_attributes(result, 2)
+        # find quadrants for each geometry
+        quads = quad_position(lisa.q)
 
-    # create weights
-    weight = pu.get_weight(result, w_type, num_ngbrs)
+        return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
 
-    # calculate LISA values
-    lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight,
-                                        permutations=permutations)
+    def local_bivariate_stat(self, subquery, attr1, attr2,
+                             permutations, geom_col, id_col,
+                             w_type, num_ngbrs):
+        """
+            Moran's I (local) Bivariate (untested)
+        """
 
-    # find clustering of significance
-    lisa_sig = quad_position(lisa.q)
+        qvals = OrderedDict([("id_col", id_col),
+                             ("attr1", attr1),
+                             ("attr2", attr2),
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
-    return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
+        query = pu.construct_neighbor_query(w_type, qvals)
+
+        result = self.query_runner.get_result(query)
+
+        # collect attributes
+        attr1_vals = pu.get_attributes(result, 1)
+        attr2_vals = pu.get_attributes(result, 2)
+
+        # create weights
+        weight = pu.get_weight(result, w_type, num_ngbrs)
+
+        # calculate LISA values
+        lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight,
+                                            permutations=permutations)
+
+        # find clustering of significance
+        lisa_sig = quad_position(lisa.q)
+
+        return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
 
 # Low level functions ----------------------------------------
 

From 224fbc2fc5f6ebe1cc671da2635dbe2d73b43288 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Sat, 19 Nov 2016 09:05:35 +0000
Subject: [PATCH 32/40] move to class based markov

---
 src/pg/sql/11_markov.sql                      |   5 +-
 .../crankshaft/space_time_dynamics/markov.py  | 177 ++++---
 .../test/test_space_time_dynamics.py          | 501 ++++++++++--------
 3 files changed, 366 insertions(+), 317 deletions(-)

diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql
index 1124abd..da02c66 100644
--- a/src/pg/sql/11_markov.sql
+++ b/src/pg/sql/11_markov.sql
@@ -22,10 +22,11 @@ CREATE OR REPLACE FUNCTION
 RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT)
 AS $$
 
-  from crankshaft.space_time_dynamics import spatial_markov_trend
+  from crankshaft.space_time_dynamics import Markov
+  markov = Markov()
 
   ## TODO: use named parameters or a dictionary
-  return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
+  return markov.spatial_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 -- input table format: identical to above but in a predictable format
diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
index ae788d7..7984e0c 100644
--- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
@@ -8,92 +8,104 @@ import pysal as ps
 import plpy
 import crankshaft.pysal_utils as pu
 
-def spatial_markov_trend(subquery, time_cols, num_classes=7,
-                         w_type='knn', num_ngbrs=5, permutations=0,
-                         geom_col='the_geom', id_col='cartodb_id'):
-    """
-        Predict the trends of a unit based on:
-        1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile)
-        2. average class of its neighbors
 
-        Inputs:
-        @param subquery string: e.g., SELECT the_geom, cartodb_id,
-          interesting_time_column FROM table_name
-        @param time_cols list of strings: list of strings of column names
-        @param num_classes (optional): number of classes to break distribution
-          of values into. Currently uses quantile bins.
-        @param w_type string (optional): weight type ('knn' or 'queen')
-        @param num_ngbrs int (optional): number of neighbors (if knn type)
-        @param permutations int (optional): number of permutations for test
-          stats
-        @param geom_col string (optional): name of column which contains the
-          geometries
-        @param id_col string (optional): name of column which has the ids of
-          the table
+class QueryRunner:
+    def get_result(self, query):
+        try:
+            data = plpy.execute(query)
 
-        Outputs:
-        @param trend_up float: probablity that a geom will move to a higher
-          class
-        @param trend_down float: probablity that a geom will move to a lower
-          class
-        @param trend float: (trend_up - trend_down) / trend_static
-        @param volatility float: a measure of the volatility based on
-          probability stddev(prob array)
-    """
+            if len(data) == 0:
+                return zip([None], [None], [None], [None], [None])
 
-    if len(time_cols) < 2:
-        plpy.error('More than one time column needs to be passed')
+            return data
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % err)
 
-    qvals = {"id_col": id_col,
-             "time_cols": time_cols,
-             "geom_col": geom_col,
-             "subquery": subquery,
-             "num_ngbrs": num_ngbrs}
 
-    try:
-        query_result = plpy.execute(
-            pu.construct_neighbor_query(w_type, qvals)
-        )
-        if len(query_result) == 0:
-            return zip([None], [None], [None], [None], [None])
-    except plpy.SPIError, e:
-        plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals)))
-        plpy.error('Analysis failed: %s' % e)
-        return zip([None], [None], [None], [None], [None])
+class Markov:
+    def __init__(self, query_runner=None):
+        if query_runner is None:
+            self.query_runner = QueryRunner()
+        else:
+            self.query_runner = query_runner
 
-    ## build weight
-    weights = pu.get_weight(query_result, w_type)
-    weights.transform = 'r'
+    def spatial_trend(self, subquery, time_cols, num_classes=7,
+                      w_type='knn', num_ngbrs=5, permutations=0,
+                      geom_col='the_geom', id_col='cartodb_id'):
+        """
+            Predict the trends of a unit based on:
+            1. history of its transitions to different classes (e.g., 1st
+               quantile -> 2nd quantile)
+            2. average class of its neighbors
 
-    ## prep time data
-    t_data = get_time_data(query_result, time_cols)
+            Inputs:
+            @param subquery string: e.g., SELECT the_geom, cartodb_id,
+              interesting_time_column FROM table_name
+            @param time_cols list of strings: list of strings of column names
+            @param num_classes (optional): number of classes to break
+              distribution of values into. Currently uses quantile bins.
+            @param w_type string (optional): weight type ('knn' or 'queen')
+            @param num_ngbrs int (optional): number of neighbors (if knn type)
+            @param permutations int (optional): number of permutations for test
+              stats
+            @param geom_col string (optional): name of column which contains
+              the geometries
+            @param id_col string (optional): name of column which has the ids
+              of the table
 
-    plpy.debug('shape of t_data %d, %d' % t_data.shape)
-    plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape)
-    plpy.debug('first num elements: %f' % t_data[0, 0])
+            Outputs:
+            @param trend_up float: probablity that a geom will move to a higher
+              class
+            @param trend_down float: probablity that a geom will move to a
+              lower class
+            @param trend float: (trend_up - trend_down) / trend_static
+            @param volatility float: a measure of the volatility based on
+              probability stddev(prob array)
+        """
 
-    sp_markov_result = ps.Spatial_Markov(t_data,
-                                         weights,
-                                         k=num_classes,
-                                         fixed=False,
-                                         permutations=permutations)
+        if len(time_cols) < 2:
+            plpy.error('More than one time column needs to be passed')
 
-    ## get lag classes
-    lag_classes = ps.Quantiles(
-        ps.lag_spatial(weights, t_data[:, -1]),
-        k=num_classes).yb
+        qvals = {"id_col": id_col,
+                 "time_cols": time_cols,
+                 "geom_col": geom_col,
+                 "subquery": subquery,
+                 "num_ngbrs": num_ngbrs}
 
-    ## look up probablity distribution for each unit according to class and lag class
-    prob_dist = get_prob_dist(sp_markov_result.P,
-                              lag_classes,
-                              sp_markov_result.classes[:, -1])
+        query = pu.construct_neighbor_query(w_type, qvals)
 
-    ## find the ups and down and overall distribution of each cell
-    trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist,
-                                                             sp_markov_result.classes[:, -1])
+        query_result = self.query_runner.get_result(query)
+
+        # build weight
+        weights = pu.get_weight(query_result, w_type)
+        weights.transform = 'r'
+
+        # prep time data
+        t_data = get_time_data(query_result, time_cols)
+
+        sp_markov_result = ps.Spatial_Markov(t_data,
+                                             weights,
+                                             k=num_classes,
+                                             fixed=False,
+                                             permutations=permutations)
+
+        # get lag classes
+        lag_classes = ps.Quantiles(
+            ps.lag_spatial(weights, t_data[:, -1]),
+            k=num_classes).yb
+
+        # look up probablity distribution for each unit according to class and
+        #  lag class
+        prob_dist = get_prob_dist(sp_markov_result.P,
+                                  lag_classes,
+                                  sp_markov_result.classes[:, -1])
+
+        # find the ups and down and overall distribution of each cell
+        trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1])
+
+        # output the results
+        return zip(trend, trend_up, trend_down, volatility, weights.id_order)
 
-    ## output the results
-    return zip(trend, trend_up, trend_down, volatility, weights.id_order)
 
 def get_time_data(markov_data, time_cols):
     """
@@ -103,7 +115,8 @@ def get_time_data(markov_data, time_cols):
     return np.array([[x['attr' + str(i)] for x in markov_data]
                      for i in range(1, num_attrs+1)], dtype=float).transpose()
 
-## not currently used
+
+# not currently used
 def rebin_data(time_data, num_time_per_bin):
     """
         Convert an n x l matrix into an (n/m) x l matrix where the values are
@@ -131,14 +144,16 @@ def rebin_data(time_data, num_time_per_bin):
     """
 
     if time_data.shape[1] % num_time_per_bin == 0:
-        ## if fit is perfect, then use it
+        # if fit is perfect, then use it
         n_max = time_data.shape[1] / num_time_per_bin
     else:
-        ## fit remainders into an additional column
+        # fit remainders into an additional column
         n_max = time_data.shape[1] / num_time_per_bin + 1
 
-    return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1)
-                     for i in range(n_max)]).T
+    return np.array(
+      [time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1)
+       for i in range(n_max)]).T
+
 
 def get_prob_dist(transition_matrix, lag_indices, unit_indices):
     """
@@ -157,6 +172,7 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices):
     return np.array([transition_matrix[(lag_indices[i], unit_indices[i])]
                      for i in range(len(lag_indices))])
 
+
 def get_prob_stats(prob_dist, unit_indices):
     """
         get the statistics of the probability distributions
@@ -179,11 +195,12 @@ def get_prob_stats(prob_dist, unit_indices):
         trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum()
         trend_down[i] = prob_dist[i, :unit_indices[i]].sum()
         if prob_dist[i, unit_indices[i]] > 0.0:
-            trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]]
+            trend[i] = (trend_up[i] - trend_down[i]) / (
+              prob_dist[i, unit_indices[i]])
         else:
             trend[i] = None
 
-    ## calculate volatility of distribution
+    # calculate volatility of distribution
     volatility = prob_dist.std(axis=1)
 
     return trend_up, trend_down, trend, volatility
diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py
index 54ffc9d..21f3afc 100644
--- a/src/py/crankshaft/test/test_space_time_dynamics.py
+++ b/src/py/crankshaft/test/test_space_time_dynamics.py
@@ -9,81 +9,100 @@ import unittest
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
+from helper import fixture_file
 
+from crankshaft.space_time_dynamics import Markov
 import crankshaft.space_time_dynamics as std
 from crankshaft import random_seeds
+from crankshaft.clustering import QueryRunner
 import json
 
+
+class FakeQueryRunner(QueryRunner):
+    def __init__(self, data):
+        self.mock_result = data
+
+    def get_result(self, query):
+        return self.mock_result
+
+
 class SpaceTimeTests(unittest.TestCase):
     """Testing class for Markov Functions."""
 
     def setUp(self):
-        plpy._reset()
+        # plpy._reset()
         self.params = {"id_col": "cartodb_id",
                        "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'],
                        "subquery": "SELECT * FROM a_list",
                        "geom_col": "the_geom",
                        "num_ngbrs": 321}
-        self.neighbors_data = json.loads(open(fixture_file('neighbors_markov.json')).read())
+        self.neighbors_data = json.loads(
+          open(fixture_file('neighbors_markov.json')).read())
         self.markov_data = json.loads(open(fixture_file('markov.json')).read())
 
-        self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T
+        self.time_data = np.array([i * np.ones(10, dtype=float)
+                                   for i in range(10)]).T
 
         self.transition_matrix = np.array([
-                [[ 0.96341463, 0.0304878 , 0.00609756, 0.        , 0.        ],
-                 [ 0.06040268, 0.83221477, 0.10738255, 0.        , 0.        ],
-                 [ 0.        , 0.14      , 0.74      , 0.12      , 0.        ],
-                 [ 0.        , 0.03571429, 0.32142857, 0.57142857, 0.07142857],
-                 [ 0.        , 0.        , 0.        , 0.16666667, 0.83333333]],
-                [[ 0.79831933, 0.16806723, 0.03361345, 0.        , 0.        ],
-                 [ 0.0754717 , 0.88207547, 0.04245283, 0.        , 0.        ],
-                 [ 0.00537634, 0.06989247, 0.8655914 , 0.05913978, 0.        ],
-                 [ 0.        , 0.        , 0.06372549, 0.90196078, 0.03431373],
-                 [ 0.        , 0.        , 0.        , 0.19444444, 0.80555556]],
-                [[ 0.84693878, 0.15306122, 0.        , 0.        , 0.        ],
-                 [ 0.08133971, 0.78947368, 0.1291866 , 0.        , 0.        ],
-                 [ 0.00518135, 0.0984456 , 0.79274611, 0.0984456 , 0.00518135],
-                 [ 0.        , 0.        , 0.09411765, 0.87058824, 0.03529412],
-                 [ 0.        , 0.        , 0.        , 0.10204082, 0.89795918]],
-                [[ 0.8852459 , 0.09836066, 0.        , 0.01639344, 0.        ],
-                 [ 0.03875969, 0.81395349, 0.13953488, 0.        , 0.00775194],
-                 [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ],
-                 [ 0.        , 0.02339181, 0.12865497, 0.75438596, 0.09356725],
-                 [ 0.        , 0.        , 0.        , 0.09661836, 0.90338164]],
-                [[ 0.33333333, 0.66666667, 0.        , 0.        , 0.        ],
-                 [ 0.0483871 , 0.77419355, 0.16129032, 0.01612903, 0.        ],
-                 [ 0.01149425, 0.16091954, 0.74712644, 0.08045977, 0.        ],
-                 [ 0.        , 0.01036269, 0.06217617, 0.89637306, 0.03108808],
-                 [ 0.        , 0.        , 0.        , 0.02352941, 0.97647059]]]
+                [[0.96341463, 0.0304878, 0.00609756, 0., 0.],
+                 [0.06040268, 0.83221477, 0.10738255, 0., 0.],
+                 [0., 0.14, 0.74, 0.12, 0.],
+                 [0., 0.03571429, 0.32142857, 0.57142857, 0.07142857],
+                 [0., 0., 0., 0.16666667, 0.83333333]],
+                [[0.79831933, 0.16806723, 0.03361345, 0., 0.],
+                 [0.0754717, 0.88207547, 0.04245283, 0., 0.],
+                 [0.00537634, 0.06989247, 0.8655914, 0.05913978, 0.],
+                 [0., 0., 0.06372549, 0.90196078, 0.03431373],
+                 [0., 0., 0., 0.19444444, 0.80555556]],
+                [[0.84693878, 0.15306122, 0., 0., 0.],
+                 [0.08133971, 0.78947368, 0.1291866, 0., 0.],
+                 [0.00518135, 0.0984456, 0.79274611, 0.0984456, 0.00518135],
+                 [0., 0., 0.09411765, 0.87058824, 0.03529412],
+                 [0., 0., 0., 0.10204082, 0.89795918]],
+                [[0.8852459, 0.09836066, 0., 0.01639344, 0.],
+                 [0.03875969, 0.81395349, 0.13953488, 0., 0.00775194],
+                 [0.0049505, 0.09405941, 0.77722772, 0.11881188, 0.0049505],
+                 [0., 0.02339181, 0.12865497, 0.75438596, 0.09356725],
+                 [0., 0., 0., 0.09661836, 0.90338164]],
+                [[0.33333333, 0.66666667, 0., 0., 0.],
+                 [0.0483871, 0.77419355, 0.16129032, 0.01612903, 0.],
+                 [0.01149425, 0.16091954, 0.74712644, 0.08045977, 0.],
+                 [0., 0.01036269, 0.06217617, 0.89637306, 0.03108808],
+                 [0., 0., 0., 0.02352941, 0.97647059]]]
                  )
 
     def test_spatial_markov(self):
         """Test Spatial Markov."""
-        data = [ { 'id': d['id'],
-                   'attr1': d['y1995'],
-                   'attr2': d['y1996'],
-                   'attr3': d['y1997'],
-                   'attr4': d['y1998'],
-                   'attr5': d['y1999'],
-                   'attr6': d['y2000'],
-                   'attr7': d['y2001'],
-                   'attr8': d['y2002'],
-                   'attr9': d['y2003'],
-                   'attr10': d['y2004'],
-                   'attr11': d['y2005'],
-                   'attr12': d['y2006'],
-                   'attr13': d['y2007'],
-                   'attr14': d['y2008'],
-                   'attr15': d['y2009'],
-                   'neighbors': d['neighbors'] } for d in self.neighbors_data]
-        print(str(data[0]))
-        plpy._define_result('select', data)
+        data = [{'id': d['id'],
+                 'attr1': d['y1995'],
+                 'attr2': d['y1996'],
+                 'attr3': d['y1997'],
+                 'attr4': d['y1998'],
+                 'attr5': d['y1999'],
+                 'attr6': d['y2000'],
+                 'attr7': d['y2001'],
+                 'attr8': d['y2002'],
+                 'attr9': d['y2003'],
+                 'attr10': d['y2004'],
+                 'attr11': d['y2005'],
+                 'attr12': d['y2006'],
+                 'attr13': d['y2007'],
+                 'attr14': d['y2008'],
+                 'attr15': d['y2009'],
+                 'neighbors': d['neighbors']} for d in self.neighbors_data]
+        # print(str(data[0]))
+        markov = Markov(FakeQueryRunner(data))
         random_seeds.set_random_seeds(1234)
 
-        result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id')
+        result = markov.spatial_trend('subquery',
+                                      ['y1995', 'y1996', 'y1997', 'y1998',
+                                       'y1999', 'y2000', 'y2001', 'y2002',
+                                       'y2003', 'y2004', 'y2005', 'y2006',
+                                       'y2007', 'y2008', 'y2009'],
+                                      5, 'knn', 5, 0, 'the_geom',
+                                      'cartodb_id')
 
-        self.assertTrue(result != None)
+        self.assertTrue(result is not None)
         result = [(row[0], row[1], row[2], row[3], row[4]) for row in result]
         print result[0]
         expected = self.markov_data
@@ -94,173 +113,178 @@ class SpaceTimeTests(unittest.TestCase):
 
     def test_get_time_data(self):
         """Test get_time_data"""
-        data = [ { 'attr1': d['y1995'],
-                   'attr2': d['y1996'],
-                   'attr3': d['y1997'],
-                   'attr4': d['y1998'],
-                   'attr5': d['y1999'],
-                   'attr6': d['y2000'],
-                   'attr7': d['y2001'],
-                   'attr8': d['y2002'],
-                   'attr9': d['y2003'],
-                   'attr10': d['y2004'],
-                   'attr11': d['y2005'],
-                   'attr12': d['y2006'],
-                   'attr13': d['y2007'],
-                   'attr14': d['y2008'],
-                   'attr15': d['y2009'] } for d in self.neighbors_data]
+        data = [{'attr1': d['y1995'],
+                 'attr2': d['y1996'],
+                 'attr3': d['y1997'],
+                 'attr4': d['y1998'],
+                 'attr5': d['y1999'],
+                 'attr6': d['y2000'],
+                 'attr7': d['y2001'],
+                 'attr8': d['y2002'],
+                 'attr9': d['y2003'],
+                 'attr10': d['y2004'],
+                 'attr11': d['y2005'],
+                 'attr12': d['y2006'],
+                 'attr13': d['y2007'],
+                 'attr14': d['y2008'],
+                 'attr15': d['y2009']} for d in self.neighbors_data]
 
-        result = std.get_time_data(data, ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'])
+        result = std.get_time_data(data, ['y1995', 'y1996', 'y1997', 'y1998',
+                                          'y1999', 'y2000', 'y2001', 'y2002',
+                                          'y2003', 'y2004', 'y2005', 'y2006',
+                                          'y2007', 'y2008', 'y2009'])
 
-        ## expected was prepared from PySAL example:
-        ### f = ps.open(ps.examples.get_path("usjoin.csv"))
-        ### pci = np.array([f.by_col[str(y)] for y in range(1995, 2010)]).transpose()
-        ### rpci = pci / (pci.mean(axis = 0))
+        # expected was prepared from PySAL example:
+        # f = ps.open(ps.examples.get_path("usjoin.csv"))
+        # pci = np.array([f.by_col[str(y)]
+        #                 for y in range(1995, 2010)]).transpose()
+        # rpci = pci / (pci.mean(axis = 0))
 
-        expected = np.array([[ 0.87654416, 0.863147, 0.85637567, 0.84811668, 0.8446154,  0.83271652
-        ,  0.83786314, 0.85012593, 0.85509656, 0.86416612, 0.87119375, 0.86302631
-        ,  0.86148267, 0.86252252, 0.86746356],
-         [ 0.9188951,  0.91757931, 0.92333258, 0.92517289, 0.92552388, 0.90746978
-        ,  0.89830489, 0.89431991, 0.88924794, 0.89815176, 0.91832091, 0.91706054
-        ,  0.90139505, 0.87897455, 0.86216858],
-         [ 0.82591007, 0.82548596, 0.81989793, 0.81503235, 0.81731522, 0.78964559
-        ,  0.80584442, 0.8084998,  0.82258551, 0.82668196, 0.82373724, 0.81814804
-        ,  0.83675961, 0.83574199, 0.84647177],
-         [ 1.09088176, 1.08537689, 1.08456418, 1.08415404, 1.09898841, 1.14506948
-        ,  1.12151133, 1.11160697, 1.10888621, 1.11399806, 1.12168029, 1.13164797
-        ,  1.12958508, 1.11371818, 1.09936775],
-         [ 1.10731446, 1.11373944, 1.13283638, 1.14472559, 1.15910025, 1.16898201
-        ,  1.17212488, 1.14752303, 1.11843284, 1.11024964, 1.11943471, 1.11736468
-        ,  1.10863242, 1.09642516, 1.07762337],
-         [ 1.42269757, 1.42118434, 1.44273502, 1.43577571, 1.44400684, 1.44184737
-        ,  1.44782832, 1.41978227, 1.39092208, 1.4059372,  1.40788646, 1.44052766
-        ,  1.45241216, 1.43306098, 1.4174431 ],
-         [ 1.13073885, 1.13110513, 1.11074708, 1.13364636, 1.13088149, 1.10888138
-        ,  1.11856629, 1.13062931, 1.11944984, 1.12446239, 1.11671008, 1.10880034
-        ,  1.08401709, 1.06959206, 1.07875225],
-         [ 1.04706124, 1.04516831, 1.04253372, 1.03239987, 1.02072545, 0.99854316
-        ,  0.9880258,  0.99669587, 0.99327676, 1.01400905, 1.03176742, 1.040511
-        ,  1.01749645, 0.9936394,  0.98279746],
-         [ 0.98996986, 1.00143564, 0.99491,  1.00188408, 1.00455845, 0.99127006
-        ,  0.97925917, 0.9683482,  0.95335147, 0.93694787, 0.94308213, 0.92232874
-        ,  0.91284091, 0.89689833, 0.88928858],
-         [ 0.87418391, 0.86416601, 0.84425695, 0.8404494,  0.83903044, 0.8578708
-        ,  0.86036185, 0.86107306, 0.8500772,  0.86981998, 0.86837929, 0.87204141
-        ,  0.86633032, 0.84946077, 0.83287146],
-         [ 1.14196118, 1.14660262, 1.14892712, 1.14909594, 1.14436624, 1.14450183
-        ,  1.12349752, 1.12596664, 1.12213996, 1.1119989,  1.10257792, 1.10491258
-        ,  1.11059842, 1.10509795, 1.10020097],
-         [ 0.97282463, 0.96700147, 0.96252588, 0.9653878,  0.96057687, 0.95831051
-        ,  0.94480909, 0.94804195, 0.95430286, 0.94103989, 0.92122519, 0.91010201
-        ,  0.89280392, 0.89298243, 0.89165385],
-         [ 0.94325468, 0.96436902, 0.96455242, 0.95243009, 0.94117647, 0.9480927
-        ,  0.93539182, 0.95388718, 0.94597005, 0.96918424, 0.94781281, 0.93466815
-        ,  0.94281559, 0.96520315, 0.96715441],
-         [ 0.97478408, 0.98169225, 0.98712809, 0.98474769, 0.98559897, 0.98687073
-        ,  0.99237486, 0.98209969, 0.9877653,  0.97399471, 0.96910087, 0.98416665
-        ,  0.98423613, 0.99823861, 0.99545704],
-         [ 0.85570269, 0.85575915, 0.85986132, 0.85693406, 0.8538012,  0.86191535
-        ,  0.84981451, 0.85472102, 0.84564835, 0.83998883, 0.83478547, 0.82803648
-        ,  0.8198736,  0.82265395, 0.8399404 ],
-         [ 0.87022047, 0.85996258, 0.85961813, 0.85689572, 0.83947136, 0.82785597
-        ,  0.86008789, 0.86776298, 0.86720209, 0.8676334,  0.89179317, 0.94202108
-        ,  0.9422231,  0.93902708, 0.94479184],
-         [ 0.90134907, 0.90407738, 0.90403991, 0.90201769, 0.90399238, 0.90906632
-        ,  0.92693339, 0.93695966, 0.94242697, 0.94338265, 0.91981796, 0.91108804
-        ,  0.90543476, 0.91737138, 0.94793657],
-         [ 1.1977611,  1.18222564, 1.18439158, 1.18267865, 1.19286723, 1.20172869
-        ,  1.21328691, 1.22624778, 1.22397075, 1.23857042, 1.24419893, 1.23929384
-        ,  1.23418676, 1.23626739, 1.26754398],
-         [ 1.24919678, 1.25754773, 1.26991161, 1.28020651, 1.30625667, 1.34790023
-        ,  1.34399863, 1.32575181, 1.30795492, 1.30544841, 1.30303302, 1.32107766
-        ,  1.32936244, 1.33001241, 1.33288462],
-         [ 1.06768004, 1.03799276, 1.03637303, 1.02768449, 1.03296093, 1.05059016
-        ,  1.03405057, 1.02747623, 1.03162734, 0.9961416,  0.97356208, 0.94241549
-        ,  0.92754547, 0.92549227, 0.92138102],
-         [ 1.09475614, 1.11526796, 1.11654299, 1.13103948, 1.13143264, 1.13889622
-        ,  1.12442212, 1.13367018, 1.13982256, 1.14029944, 1.11979401, 1.10905389
-        ,  1.10577769, 1.11166825, 1.09985155],
-         [ 0.76530058, 0.76612841, 0.76542451, 0.76722683, 0.76014284, 0.74480073
-        ,  0.76098396, 0.76156903, 0.76651952, 0.76533288, 0.78205934, 0.76842416
-        ,  0.77487118, 0.77768683, 0.78801192],
-         [ 0.98391336, 0.98075816, 0.98295341, 0.97386015, 0.96913803, 0.97370819
-        ,  0.96419154, 0.97209861, 0.97441313, 0.96356162, 0.94745352, 0.93965462
-        ,  0.93069645, 0.94020973, 0.94358232],
-         [ 0.83561828, 0.82298088, 0.81738502, 0.81748588, 0.80904801, 0.80071489
-        ,  0.83358256, 0.83451613, 0.85175032, 0.85954307, 0.86790024, 0.87170334
-        ,  0.87863799, 0.87497981, 0.87888675],
-         [ 0.98845573, 1.02092428, 0.99665283, 0.99141823, 0.99386619, 0.98733195
-        ,  0.99644997, 0.99669587, 1.02559097, 1.01116651, 0.99988024, 0.97906749
-        ,  0.99323123, 1.00204939, 0.99602148],
-         [ 1.14930913, 1.15241949, 1.14300962, 1.14265542, 1.13984683, 1.08312397
-        ,  1.05192626, 1.04230892, 1.05577278, 1.08569751, 1.12443486, 1.08891079
-        ,  1.08603695, 1.05997314, 1.02160943],
-         [ 1.11368269, 1.1057147,  1.11893431, 1.13778669, 1.1432272,  1.18257029
-        ,  1.16226243, 1.16009196, 1.14467789, 1.14820235, 1.12386598, 1.12680236
-        ,  1.12357937, 1.1159258,  1.12570828],
-         [ 1.30379431, 1.30752186, 1.31206366, 1.31532267, 1.30625667, 1.31210239
-        ,  1.29989156, 1.29203193, 1.27183516, 1.26830786, 1.2617743,  1.28656675
-        ,  1.29734097, 1.29390205, 1.29345446],
-         [ 0.83953719, 0.82701448, 0.82006005, 0.81188876, 0.80294864, 0.78772975
-        ,  0.82848011, 0.8259679,  0.82435705, 0.83108634, 0.84373784, 0.83891093
-        ,  0.84349247, 0.85637272, 0.86539395],
-         [ 1.23450087, 1.2426022,  1.23537935, 1.23581293, 1.24522626, 1.2256767
-        ,  1.21126648, 1.19377804, 1.18355337, 1.19674434, 1.21536573, 1.23653297
-        ,  1.27962009, 1.27968392, 1.25907738],
-         [ 0.9769662,  0.97400719, 0.98035944, 0.97581531, 0.95543282, 0.96480308
-        ,  0.94686376, 0.93679073, 0.92540049, 0.92988835, 0.93442917, 0.92100464
-        ,  0.91475304, 0.90249622, 0.9021363 ],
-         [ 0.84986886, 0.8986851,  0.84295997, 0.87280534, 0.85659368, 0.88937573
-        ,  0.894401, 0.90448993, 0.95495898, 0.92698333, 0.94745352, 0.92562488
-        ,  0.96635366, 1.02520312, 1.0394296 ],
-         [ 1.01922808, 1.00258203, 1.00974428, 1.00303417, 0.99765073, 1.00759019
-        ,  0.99192968, 0.99747298, 0.99550759, 0.97583768, 0.9610168,  0.94779638
-        ,  0.93759089, 0.93353431, 0.94121705],
-         [ 0.86367411, 0.85558932, 0.85544346, 0.85103025, 0.84336613, 0.83434854
-        ,  0.85813595, 0.84667961, 0.84374558, 0.85951183, 0.87194227, 0.89455097
-        ,  0.88283929, 0.90349491, 0.90600675],
-         [ 1.00947534, 1.00411055, 1.00698819, 0.99513687, 0.99291086, 1.00581626
-        ,  0.98850522, 0.99291168, 0.98983209, 0.97511924, 0.96134615, 0.96382634
-        ,  0.95011401, 0.9434686,  0.94637765],
-         [ 1.05712571, 1.05459419, 1.05753012, 1.04880786, 1.05103857, 1.04800023
-        ,  1.03024941, 1.04200483, 1.0402554,  1.03296979, 1.02191682, 1.02476275
-        ,  1.02347523, 1.02517684, 1.04359571],
-         [ 1.07084189, 1.06669497, 1.07937623, 1.07387988, 1.0794043,  1.0531801
-        ,  1.07452771, 1.09383478, 1.1052447,  1.10322136, 1.09167939, 1.08772756
-        ,  1.08859544, 1.09177338, 1.1096083 ],
-         [ 0.86719222, 0.86628896, 0.86675156, 0.86425632, 0.86511809, 0.86287327
-        ,  0.85169796, 0.85411285, 0.84886336, 0.84517414, 0.84843858, 0.84488343
-        ,  0.83374329, 0.82812044, 0.82878599],
-         [ 0.88389211, 0.92288667, 0.90282398, 0.91229186, 0.92023286, 0.92652175
-        ,  0.94278865, 0.93682452, 0.98655146, 0.992237, 0.9798497,  0.93869677
-        ,  0.96947771, 1.00362626, 0.98102351],
-         [ 0.97082064, 0.95320233, 0.94534081, 0.94215593, 0.93967,  0.93092109
-        ,  0.92662519, 0.93412152, 0.93501274, 0.92879506, 0.92110542, 0.91035556
-        ,  0.90430364, 0.89994694, 0.90073864],
-         [ 0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824, 0.98882205
-        ,  0.97662234, 0.95601578, 0.94905385, 0.94934888, 0.97152609, 0.97163004
-        ,  0.9700702,  0.97158948, 0.95884908],
-         [ 0.83980439, 0.84726737, 0.85747,  0.85467221, 0.8556751,  0.84818516
-        ,  0.85265681, 0.84502402, 0.82645665, 0.81743586, 0.83550406, 0.83338919
-        ,  0.83511679, 0.82136617, 0.80921874],
-         [ 0.95118156, 0.9466212,  0.94688098, 0.9508583,  0.9512441,  0.95440787
-        ,  0.96364363, 0.96804412, 0.97136214, 0.97583768, 0.95571724, 0.96895368
-        ,  0.97001634, 0.97082733, 0.98782366],
-         [ 1.08910044, 1.08248968, 1.08492895, 1.08656923, 1.09454249, 1.10558188
-        ,  1.1214086,  1.12292577, 1.13021031, 1.13342735, 1.14686068, 1.14502975
-        ,  1.14474747, 1.14084037, 1.16142926],
-         [ 1.06336033, 1.07365823, 1.08691496, 1.09764846, 1.11669863, 1.11856702
-        ,  1.09764283, 1.08815849, 1.08044313, 1.09278827, 1.07003204, 1.08398066
-        ,  1.09831768, 1.09298232, 1.09176125],
-         [ 0.79772065, 0.78829196, 0.78581151, 0.77615922, 0.77035744, 0.77751194
-        ,  0.79902974, 0.81437881, 0.80788828, 0.79603865, 0.78966436, 0.79949807
-        ,  0.80172182, 0.82168155, 0.85587911],
-         [ 1.0052447,  1.00007696, 1.00475899, 1.00613942, 1.00639561, 1.00162979
-        ,  0.99860739, 1.00814981, 1.00574316, 0.99030032, 0.97682565, 0.97292596
-        ,  0.96519561, 0.96173403, 0.95890284],
-         [ 0.95808419, 0.9382568,  0.9654441,  0.95561201, 0.96987289, 0.96608031
-        ,  0.99727185, 1.00781194, 1.03484236, 1.05333619, 1.0983263,  1.1704974
-        ,  1.17025154, 1.18730553, 1.14242645]])
+        expected = np.array(
+          [[0.87654416, 0.863147, 0.85637567, 0.84811668, 0.8446154,
+            0.83271652,  0.83786314, 0.85012593, 0.85509656, 0.86416612,
+            0.87119375, 0.86302631,  0.86148267, 0.86252252, 0.86746356],
+           [0.9188951,  0.91757931, 0.92333258, 0.92517289, 0.92552388,
+            0.90746978,  0.89830489, 0.89431991, 0.88924794, 0.89815176,
+            0.91832091, 0.91706054,  0.90139505, 0.87897455, 0.86216858],
+           [0.82591007, 0.82548596, 0.81989793, 0.81503235, 0.81731522,
+            0.78964559,  0.80584442, 0.8084998,  0.82258551, 0.82668196,
+            0.82373724, 0.81814804,  0.83675961, 0.83574199, 0.84647177],
+           [1.09088176, 1.08537689, 1.08456418, 1.08415404, 1.09898841,
+            1.14506948,  1.12151133, 1.11160697, 1.10888621, 1.11399806,
+            1.12168029, 1.13164797,  1.12958508, 1.11371818, 1.09936775],
+           [1.10731446, 1.11373944, 1.13283638, 1.14472559, 1.15910025,
+            1.16898201,  1.17212488, 1.14752303, 1.11843284, 1.11024964,
+            1.11943471, 1.11736468,  1.10863242, 1.09642516, 1.07762337],
+           [1.42269757, 1.42118434, 1.44273502, 1.43577571, 1.44400684,
+            1.44184737,  1.44782832, 1.41978227, 1.39092208, 1.4059372,
+            1.40788646, 1.44052766,  1.45241216, 1.43306098, 1.4174431],
+           [1.13073885, 1.13110513, 1.11074708, 1.13364636, 1.13088149,
+            1.10888138,  1.11856629, 1.13062931, 1.11944984, 1.12446239,
+            1.11671008, 1.10880034,  1.08401709, 1.06959206, 1.07875225],
+           [1.04706124, 1.04516831, 1.04253372, 1.03239987, 1.02072545,
+            0.99854316,  0.9880258,  0.99669587, 0.99327676, 1.01400905,
+            1.03176742, 1.040511,  1.01749645, 0.9936394,  0.98279746],
+           [0.98996986, 1.00143564, 0.99491,  1.00188408, 1.00455845,
+            0.99127006,  0.97925917, 0.9683482,  0.95335147, 0.93694787,
+            0.94308213, 0.92232874,  0.91284091, 0.89689833, 0.88928858],
+           [0.87418391, 0.86416601, 0.84425695, 0.8404494,  0.83903044,
+            0.8578708,  0.86036185, 0.86107306, 0.8500772,  0.86981998,
+            0.86837929, 0.87204141,  0.86633032, 0.84946077, 0.83287146],
+           [1.14196118, 1.14660262, 1.14892712, 1.14909594, 1.14436624,
+            1.14450183,  1.12349752, 1.12596664, 1.12213996, 1.1119989,
+            1.10257792, 1.10491258,  1.11059842, 1.10509795, 1.10020097],
+           [0.97282463, 0.96700147, 0.96252588, 0.9653878,  0.96057687,
+            0.95831051,  0.94480909, 0.94804195, 0.95430286, 0.94103989,
+            0.92122519, 0.91010201,  0.89280392, 0.89298243, 0.89165385],
+           [0.94325468, 0.96436902, 0.96455242, 0.95243009, 0.94117647,
+            0.9480927,  0.93539182, 0.95388718, 0.94597005, 0.96918424,
+            0.94781281, 0.93466815,  0.94281559, 0.96520315, 0.96715441],
+           [0.97478408, 0.98169225, 0.98712809, 0.98474769, 0.98559897,
+            0.98687073,  0.99237486, 0.98209969, 0.9877653,  0.97399471,
+            0.96910087, 0.98416665,  0.98423613, 0.99823861, 0.99545704],
+           [0.85570269, 0.85575915, 0.85986132, 0.85693406, 0.8538012,
+            0.86191535,  0.84981451, 0.85472102, 0.84564835, 0.83998883,
+            0.83478547, 0.82803648,  0.8198736,  0.82265395, 0.8399404],
+           [0.87022047, 0.85996258, 0.85961813, 0.85689572, 0.83947136,
+            0.82785597,  0.86008789, 0.86776298, 0.86720209, 0.8676334,
+            0.89179317, 0.94202108,  0.9422231,  0.93902708, 0.94479184],
+           [0.90134907, 0.90407738, 0.90403991, 0.90201769, 0.90399238,
+            0.90906632,  0.92693339, 0.93695966, 0.94242697, 0.94338265,
+            0.91981796, 0.91108804,  0.90543476, 0.91737138, 0.94793657],
+           [1.1977611,  1.18222564, 1.18439158, 1.18267865, 1.19286723,
+            1.20172869,  1.21328691, 1.22624778, 1.22397075, 1.23857042,
+            1.24419893, 1.23929384,  1.23418676, 1.23626739, 1.26754398],
+           [1.24919678, 1.25754773, 1.26991161, 1.28020651, 1.30625667,
+            1.34790023,  1.34399863, 1.32575181, 1.30795492, 1.30544841,
+            1.30303302, 1.32107766,  1.32936244, 1.33001241, 1.33288462],
+           [1.06768004, 1.03799276, 1.03637303, 1.02768449, 1.03296093,
+            1.05059016,  1.03405057, 1.02747623, 1.03162734, 0.9961416,
+            0.97356208, 0.94241549,  0.92754547, 0.92549227, 0.92138102],
+           [1.09475614, 1.11526796, 1.11654299, 1.13103948, 1.13143264,
+            1.13889622,  1.12442212, 1.13367018, 1.13982256, 1.14029944,
+            1.11979401, 1.10905389,  1.10577769, 1.11166825, 1.09985155],
+           [0.76530058, 0.76612841, 0.76542451, 0.76722683, 0.76014284,
+            0.74480073,  0.76098396, 0.76156903, 0.76651952, 0.76533288,
+            0.78205934, 0.76842416,  0.77487118, 0.77768683, 0.78801192],
+           [0.98391336, 0.98075816, 0.98295341, 0.97386015, 0.96913803,
+            0.97370819,  0.96419154, 0.97209861, 0.97441313, 0.96356162,
+            0.94745352, 0.93965462,  0.93069645, 0.94020973, 0.94358232],
+           [0.83561828, 0.82298088, 0.81738502, 0.81748588, 0.80904801,
+            0.80071489,  0.83358256, 0.83451613, 0.85175032, 0.85954307,
+            0.86790024, 0.87170334,  0.87863799, 0.87497981, 0.87888675],
+           [0.98845573, 1.02092428, 0.99665283, 0.99141823, 0.99386619,
+            0.98733195,  0.99644997, 0.99669587, 1.02559097, 1.01116651,
+            0.99988024, 0.97906749,  0.99323123, 1.00204939, 0.99602148],
+           [1.14930913, 1.15241949, 1.14300962, 1.14265542, 1.13984683,
+            1.08312397,  1.05192626, 1.04230892, 1.05577278, 1.08569751,
+            1.12443486, 1.08891079,  1.08603695, 1.05997314, 1.02160943],
+           [1.11368269, 1.1057147,  1.11893431, 1.13778669, 1.1432272,
+            1.18257029,  1.16226243, 1.16009196, 1.14467789, 1.14820235,
+            1.12386598, 1.12680236,  1.12357937, 1.1159258,  1.12570828],
+           [1.30379431, 1.30752186, 1.31206366, 1.31532267, 1.30625667,
+            1.31210239,  1.29989156, 1.29203193, 1.27183516, 1.26830786,
+            1.2617743,  1.28656675,  1.29734097, 1.29390205, 1.29345446],
+           [0.83953719, 0.82701448, 0.82006005, 0.81188876, 0.80294864,
+            0.78772975,  0.82848011, 0.8259679,  0.82435705, 0.83108634,
+            0.84373784, 0.83891093,  0.84349247, 0.85637272, 0.86539395],
+           [1.23450087, 1.2426022,  1.23537935, 1.23581293, 1.24522626,
+            1.2256767,  1.21126648, 1.19377804, 1.18355337, 1.19674434,
+            1.21536573, 1.23653297,  1.27962009, 1.27968392, 1.25907738],
+           [0.9769662,  0.97400719, 0.98035944, 0.97581531, 0.95543282,
+            0.96480308,  0.94686376, 0.93679073, 0.92540049, 0.92988835,
+            0.93442917, 0.92100464,  0.91475304, 0.90249622, 0.9021363],
+           [0.84986886, 0.8986851,  0.84295997, 0.87280534, 0.85659368,
+            0.88937573,  0.894401, 0.90448993, 0.95495898, 0.92698333,
+            0.94745352, 0.92562488,  0.96635366, 1.02520312, 1.0394296],
+           [1.01922808, 1.00258203, 1.00974428, 1.00303417, 0.99765073,
+            1.00759019,  0.99192968, 0.99747298, 0.99550759, 0.97583768,
+            0.9610168,  0.94779638,  0.93759089, 0.93353431, 0.94121705],
+           [0.86367411, 0.85558932, 0.85544346, 0.85103025, 0.84336613,
+            0.83434854,  0.85813595, 0.84667961, 0.84374558, 0.85951183,
+            0.87194227, 0.89455097,  0.88283929, 0.90349491, 0.90600675],
+           [1.00947534, 1.00411055, 1.00698819, 0.99513687, 0.99291086,
+            1.00581626,  0.98850522, 0.99291168, 0.98983209, 0.97511924,
+            0.96134615, 0.96382634,  0.95011401, 0.9434686,  0.94637765],
+           [1.05712571, 1.05459419, 1.05753012, 1.04880786, 1.05103857,
+            1.04800023,  1.03024941, 1.04200483, 1.0402554,  1.03296979,
+            1.02191682, 1.02476275,  1.02347523, 1.02517684, 1.04359571],
+           [1.07084189, 1.06669497, 1.07937623, 1.07387988, 1.0794043,
+            1.0531801,  1.07452771, 1.09383478, 1.1052447,  1.10322136,
+            1.09167939, 1.08772756,  1.08859544, 1.09177338, 1.1096083],
+           [0.86719222, 0.86628896, 0.86675156, 0.86425632, 0.86511809,
+            0.86287327,  0.85169796, 0.85411285, 0.84886336, 0.84517414,
+            0.84843858, 0.84488343,  0.83374329, 0.82812044, 0.82878599],
+           [0.88389211, 0.92288667, 0.90282398, 0.91229186, 0.92023286,
+            0.92652175,  0.94278865, 0.93682452, 0.98655146, 0.992237,
+            0.9798497,  0.93869677,  0.96947771, 1.00362626, 0.98102351],
+           [0.97082064, 0.95320233, 0.94534081, 0.94215593, 0.93967,
+            0.93092109,  0.92662519, 0.93412152, 0.93501274, 0.92879506,
+            0.92110542, 0.91035556,  0.90430364, 0.89994694, 0.90073864],
+           [0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824,
+            0.98882205,  0.97662234, 0.95601578, 0.94905385, 0.94934888,
+            0.97152609, 0.97163004,  0.9700702,  0.97158948, 0.95884908],
+           [0.83980439, 0.84726737, 0.85747,  0.85467221, 0.8556751,
+            0.84818516,  0.85265681, 0.84502402, 0.82645665, 0.81743586,
+            0.83550406, 0.83338919,  0.83511679, 0.82136617, 0.80921874],
+           [0.95118156, 0.9466212,  0.94688098, 0.9508583,  0.9512441,
+            0.95440787,  0.96364363, 0.96804412, 0.97136214, 0.97583768,
+            0.95571724, 0.96895368,  0.97001634, 0.97082733, 0.98782366],
+           [1.08910044, 1.08248968, 1.08492895, 1.08656923, 1.09454249,
+            1.10558188,  1.1214086,  1.12292577, 1.13021031, 1.13342735,
+            1.14686068, 1.14502975,  1.14474747, 1.14084037, 1.16142926],
+           [1.06336033, 1.07365823, 1.08691496, 1.09764846, 1.11669863,
+            1.11856702,  1.09764283, 1.08815849, 1.08044313, 1.09278827,
+            1.07003204, 1.08398066,  1.09831768, 1.09298232, 1.09176125],
+           [0.79772065, 0.78829196, 0.78581151, 0.77615922, 0.77035744,
+            0.77751194,  0.79902974, 0.81437881, 0.80788828, 0.79603865,
+            0.78966436, 0.79949807,  0.80172182, 0.82168155, 0.85587911],
+           [1.0052447,  1.00007696, 1.00475899, 1.00613942, 1.00639561,
+            1.00162979,  0.99860739, 1.00814981, 1.00574316, 0.99030032,
+            0.97682565, 0.97292596,  0.96519561, 0.96173403, 0.95890284],
+           [0.95808419, 0.9382568,  0.9654441,  0.95561201, 0.96987289,
+            0.96608031,  0.99727185, 1.00781194, 1.03484236, 1.05333619,
+            1.0983263,  1.1704974,  1.17025154, 1.18730553, 1.14242645]])
 
         self.assertTrue(np.allclose(result, expected))
         self.assertTrue(type(result) == type(expected))
@@ -268,32 +292,35 @@ class SpaceTimeTests(unittest.TestCase):
 
     def test_rebin_data(self):
         """Test rebin_data"""
-        ## sample in double the time (even case since 10 % 2 = 0):
-        ##   (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2
-        ## = 0.5,     2.5,     4.5,     6.5,     8.5
+        # sample in double the time (even case since 10 % 2 = 0):
+        #   (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2
+        # = 0.5,     2.5,     4.5,     6.5,     8.5
         ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float)
                              for i in range(0, 10, 2)]).T
 
-        self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even))
+        self.assertTrue(
+          np.array_equal(std.rebin_data(self.time_data, 2), ans_even))
 
-        ## sample in triple the time (uneven since 10 % 3 = 1):
-        ##   (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1
-        ## = 1,         4,         7,         9
-        ans_odd  = np.array([i * np.ones(10, dtype=float)
-                             for i in (1, 4, 7, 9)]).T
-        self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd))
+        # sample in triple the time (uneven since 10 % 3 = 1):
+        #   (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1
+        # = 1,         4,         7,         9
+        ans_odd = np.array([i * np.ones(10, dtype=float)
+                            for i in (1, 4, 7, 9)]).T
+        self.assertTrue(
+          np.array_equal(std.rebin_data(self.time_data, 3), ans_odd))
 
     def test_get_prob_dist(self):
         """Test get_prob_dist"""
         lag_indices = np.array([1, 2, 3, 4])
         unit_indices = np.array([1, 3, 2, 4])
         answer = np.array([
-            [ 0.0754717 , 0.88207547, 0.04245283, 0.        , 0.        ],
-            [ 0.        , 0.        , 0.09411765, 0.87058824, 0.03529412],
-            [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ],
-            [ 0.        , 0.        , 0.        , 0.02352941, 0.97647059]
+            [0.0754717, 0.88207547, 0.04245283, 0., 0.],
+            [0., 0., 0.09411765, 0.87058824, 0.03529412],
+            [0.0049505, 0.09405941, 0.77722772, 0.11881188, 0.0049505],
+            [0., 0., 0., 0.02352941, 0.97647059]
         ])
-        result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices)
+        result = std.get_prob_dist(self.transition_matrix,
+                                   lag_indices, unit_indices)
 
         self.assertTrue(np.array_equal(result, answer))
 
@@ -301,16 +328,20 @@ class SpaceTimeTests(unittest.TestCase):
         """Test get_prob_stats"""
 
         probs = np.array([
-            [ 0.0754717 , 0.88207547, 0.04245283, 0.        , 0.        ],
-            [ 0.        , 0.        , 0.09411765, 0.87058824, 0.03529412],
-            [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ],
-            [ 0.        , 0.        , 0.        , 0.02352941, 0.97647059]
+            [0.0754717, 0.88207547, 0.04245283, 0., 0.],
+            [0., 0., 0.09411765, 0.87058824, 0.03529412],
+            [0.0049505, 0.09405941, 0.77722772, 0.11881188, 0.0049505],
+            [0., 0., 0., 0.02352941, 0.97647059]
         ])
         unit_indices = np.array([1, 3, 2, 4])
         answer_up = np.array([0.04245283, 0.03529412, 0.12376238, 0.])
         answer_down = np.array([0.0754717, 0.09411765, 0.0990099, 0.02352941])
-        answer_trend = np.array([-0.03301887 / 0.88207547, -0.05882353 / 0.87058824,  0.02475248 / 0.77722772, -0.02352941 / 0.97647059])
-        answer_volatility = np.array([ 0.34221495,  0.33705421,  0.29226542,  0.38834223])
+        answer_trend = np.array([-0.03301887 / 0.88207547,
+                                 -0.05882353 / 0.87058824,
+                                 0.02475248 / 0.77722772,
+                                 -0.02352941 / 0.97647059])
+        answer_volatility = np.array([0.34221495,  0.33705421,
+                                      0.29226542,  0.38834223])
 
         result = std.get_prob_stats(probs, unit_indices)
         result_up = result[0]

From c8f5448b7c263fce8dfd7e05b68b67bfa95e4c7d Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Sat, 19 Nov 2016 14:20:06 +0000
Subject: [PATCH 33/40] seprates out query runner

---
 src/py/crankshaft/crankshaft/__init__.py      |  1 +
 .../crankshaft/clustering/kmeans.py           | 35 ++-------------
 .../crankshaft/crankshaft/clustering/moran.py | 20 +++------
 src/py/crankshaft/crankshaft/query_runner.py  | 43 +++++++++++++++++++
 .../crankshaft/space_time_dynamics/markov.py  | 16 +------
 .../crankshaft/test/test_clustering_kmeans.py |  6 +--
 .../crankshaft/test/test_clustering_moran.py  | 17 ++------
 .../test/test_space_time_dynamics.py          | 10 +----
 8 files changed, 64 insertions(+), 84 deletions(-)
 create mode 100644 src/py/crankshaft/crankshaft/query_runner.py

diff --git a/src/py/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py
index 4e06bc5..a03b040 100644
--- a/src/py/crankshaft/crankshaft/__init__.py
+++ b/src/py/crankshaft/crankshaft/__init__.py
@@ -3,3 +3,4 @@ import crankshaft.random_seeds
 import crankshaft.clustering
 import crankshaft.space_time_dynamics
 import crankshaft.segmentation
+import query_runner
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 48b9bd3..06c6527 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -1,36 +1,7 @@
 from sklearn.cluster import KMeans
-import plpy
 import numpy as np
 
-
-class QueryRunner:
-    def get_moran(self, query):
-        """fetch data for moran's i analyses"""
-        try:
-            result = plpy.execute(query)
-            # if there are no neighbors, exit
-            if len(result) == 0:
-                return pu.empty_zipped_array(2)
-        except plpy.SPIError, e:
-            plpy.error('Analysis failed: %s' % e)
-            return pu.empty_zipped_array(2)
-
-    def get_columns(self, query, standarize):
-        """fetch data for non-spatial kmeans"""
-        try:
-            db_resp = plpy.execute(query)
-        except plpy.SPIError, err:
-            plpy.error('Analysis failed: %s' % err)
-
-        return db_resp
-
-    def get_result(self, query):
-        """fetch data for spatial kmeans"""
-        try:
-            data = plpy.execute(query)
-        except plpy.SPIError, err:
-            plpy.error("Analysis failed: %s" % err)
-        return data
+from crankshaft.query_runner import QueryRunner
 
 
 class Kmeans:
@@ -52,7 +23,7 @@ class Kmeans:
                       "FROM ({query}) As a "
                       "WHERE the_geom IS NOT NULL").format(query=query)
 
-        data = self.query_runner.get_result(full_query)
+        data = self.query_runner.get_spatial_kmeans(full_query)
 
         # Unpack query response
         xs = data[0]['xs']
@@ -92,7 +63,7 @@ class Kmeans:
                    cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
                                    for idx, val in enumerate(colnames)]))
 
-        db_resp = self.query_runner.get_columns(full_query, standarize)
+        db_resp = self.query_runner.get_nonspatial_kmeans(full_query, standarize)
 
         # fill array with values for k-means clustering
         if standarize:
diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py
index ee82932..d2c99d6 100644
--- a/src/py/crankshaft/crankshaft/clustering/moran.py
+++ b/src/py/crankshaft/crankshaft/clustering/moran.py
@@ -8,6 +8,7 @@ Moran's I geostatistics (global clustering & outliers presence)
 import pysal as ps
 import plpy
 from collections import OrderedDict
+from crankshaft.query_runner import QueryRunner
 
 # crankshaft module
 import crankshaft.pysal_utils as pu
@@ -15,15 +16,6 @@ import crankshaft.pysal_utils as pu
 # High level interface ---------------------------------------
 
 
-class QueryRunner:
-    def get_result(self, query):
-        try:
-            data = plpy.execute(query)
-        except plpy.SPIError, err:
-            plpy.error("k-means (spatial) cluster analysis failed: %s" % err)
-        return data
-
-
 class Moran:
     def __init__(self, query_runner=None):
         if query_runner is None:
@@ -47,7 +39,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_result(query)
+        result = self.query_runner.get_moran(query)
 
         # collect attributes
         attr_vals = pu.get_attributes(result)
@@ -79,7 +71,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_result(query)
+        result = self.query_runner.get_moran(query)
 
         attr_vals = pu.get_attributes(result)
         weight = pu.get_weight(result, w_type, num_ngbrs)
@@ -108,7 +100,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_result(query)
+        result = self.query_runner.get_moran(query)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -140,7 +132,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_result(query)
+        result = self.query_runner.get_moran(query)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -173,7 +165,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_result(query)
+        result = self.query_runner.get_moran(query)
 
         # collect attributes
         attr1_vals = pu.get_attributes(result, 1)
diff --git a/src/py/crankshaft/crankshaft/query_runner.py b/src/py/crankshaft/crankshaft/query_runner.py
new file mode 100644
index 0000000..5775e72
--- /dev/null
+++ b/src/py/crankshaft/crankshaft/query_runner.py
@@ -0,0 +1,43 @@
+"""class for fetching data"""
+import plpy
+
+
+class QueryRunner:
+    def get_markov(self, query):
+        try:
+            data = plpy.execute(query)
+
+            if len(data) == 0:
+                return pu.empty_zipped_array(4)
+
+            return data
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % err)
+
+    def get_moran(self, query):
+        """fetch data for moran's i analyses"""
+        try:
+            data = plpy.execute(query)
+            # if there are no neighbors, exit
+            if len(data) == 0:
+                return pu.empty_zipped_array(2)
+            return data
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % e)
+            return pu.empty_zipped_array(2)
+
+    def get_nonspatial_kmeans(self, query):
+        """fetch data for non-spatial kmeans"""
+        try:
+            data = plpy.execute(query)
+            return data
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % err)
+
+    def get_spatial_kmeans(self, query):
+        """fetch data for spatial kmeans"""
+        try:
+            data = plpy.execute(query)
+            return data
+        except plpy.SPIError, err:
+            plpy.error("Analysis failed: %s" % err)
diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
index 7984e0c..ea8dd32 100644
--- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
@@ -7,19 +7,7 @@ import numpy as np
 import pysal as ps
 import plpy
 import crankshaft.pysal_utils as pu
-
-
-class QueryRunner:
-    def get_result(self, query):
-        try:
-            data = plpy.execute(query)
-
-            if len(data) == 0:
-                return zip([None], [None], [None], [None], [None])
-
-            return data
-        except plpy.SPIError, err:
-            plpy.error('Analysis failed: %s' % err)
+from crankshaft.query_runner import QueryRunner
 
 
 class Markov:
@@ -74,7 +62,7 @@ class Markov:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        query_result = self.query_runner.get_result(query)
+        query_result = self.query_runner.get_markov(query)
 
         # build weight
         weights = pu.get_weight(query_result, w_type)
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 830ee9d..9fd2504 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -9,7 +9,7 @@ import numpy as np
 # sys.modules['plpy'] = plpy
 from helper import fixture_file
 from crankshaft.clustering import Kmeans
-from crankshaft.clustering import QueryRunner
+from crankshaft.query_runner import QueryRunner
 import crankshaft.clustering as cc
 
 from crankshaft import random_seeds
@@ -21,10 +21,10 @@ class FakeQueryRunner(QueryRunner):
     def __init__(self, mocked_result):
         self.mocked_result = mocked_result
 
-    def get_result(self, query):
+    def get_spatial_kmeans(self, query):
         return self.mocked_result
 
-    def get_columns(self, query, standarize):
+    def get_nonspatial_kmeans(self, query, standarize):
         return self.mocked_result
 
 
diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py
index 0a320fb..37cf7d0 100644
--- a/src/py/crankshaft/test/test_clustering_moran.py
+++ b/src/py/crankshaft/test/test_clustering_moran.py
@@ -1,12 +1,6 @@
 import unittest
 import numpy as np
 
-
-# from mock_plpy import MockPlPy
-# plpy = MockPlPy()
-#
-# import sys
-# sys.modules['plpy'] = plpy
 from helper import fixture_file
 from crankshaft.clustering import Moran
 from crankshaft.clustering import QueryRunner
@@ -17,14 +11,11 @@ from collections import OrderedDict
 
 
 class FakeQueryRunner(QueryRunner):
-    def __init__(self, mocked_result):
-        self.mocked_result = mocked_result
+    def __init__(self, mock_data):
+        self.mock_result = mock_data
 
-    def get_result(self, query):
-        return self.mocked_result
-
-    def get_columns(self, query):
-        return self.mocked_result
+    def get_moran(self, query):
+        return self.mock_result
 
 
 class MoranTest(unittest.TestCase):
diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py
index 21f3afc..e58c7d4 100644
--- a/src/py/crankshaft/test/test_space_time_dynamics.py
+++ b/src/py/crankshaft/test/test_space_time_dynamics.py
@@ -4,17 +4,12 @@ import numpy as np
 import unittest
 
 
-# from mock_plpy import MockPlPy
-# plpy = MockPlPy()
-#
-# import sys
-# sys.modules['plpy'] = plpy
 from helper import fixture_file
 
 from crankshaft.space_time_dynamics import Markov
 import crankshaft.space_time_dynamics as std
 from crankshaft import random_seeds
-from crankshaft.clustering import QueryRunner
+from crankshaft.query_runner import QueryRunner
 import json
 
 
@@ -22,7 +17,7 @@ class FakeQueryRunner(QueryRunner):
     def __init__(self, data):
         self.mock_result = data
 
-    def get_result(self, query):
+    def get_markov(self, query):
         return self.mock_result
 
 
@@ -30,7 +25,6 @@ class SpaceTimeTests(unittest.TestCase):
     """Testing class for Markov Functions."""
 
     def setUp(self):
-        # plpy._reset()
         self.params = {"id_col": "cartodb_id",
                        "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'],
                        "subquery": "SELECT * FROM a_list",

From 2f27622a6dbbe44d068bdbf7c1672d74112399b7 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Mon, 21 Nov 2016 16:19:54 +0000
Subject: [PATCH 34/40] strips out kmeans non spatial

---
 src/pg/sql/11_kmeans.sql                      | 18 -----
 .../crankshaft/clustering/kmeans.py           | 79 -------------------
 .../crankshaft/test/test_clustering_kmeans.py | 33 --------
 3 files changed, 130 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 1dc6d00..0899e81 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -9,24 +9,6 @@ RETURNS table (cartodb_id integer, cluster_no integer) as $$
 
 $$ LANGUAGE plpythonu;
 
--- Non-spatial k-means clustering
--- query: sql query to retrieve all the needed data
-
-CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
-  query TEXT,
-  colnames TEXT[],
-  num_clusters INTEGER,
-  id_colname TEXT DEFAULT 'cartodb_id',
-  standarize BOOLEAN DEFAULT true
-)
-RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
-
-    from crankshaft.clustering import Kmeans
-    kmeans = Kmeans()
-    return kmeans.nonspatial(query, colnames, num_clusters,
-                             id_colname, standarize)
-$$ LANGUAGE plpythonu;
-
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)
 RETURNS Numeric[] AS
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 06c6527..200dc52 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -33,82 +33,3 @@ class Kmeans:
         km = KMeans(n_clusters=no_clusters, n_init=no_init)
         labels = km.fit_predict(zip(xs, ys))
         return zip(ids, labels)
-
-    def nonspatial(self, query, colnames, num_clusters=5,
-                   id_col='cartodb_id', standarize=True):
-        """
-            query (string): A SQL query to retrieve the data required to do the
-                            k-means clustering analysis, like so:
-                            SELECT * FROM iris_flower_data
-            colnames (list): a list of the column names which contain the data
-                             of interest, like so: ["sepal_width",
-                                                    "petal_width",
-                                                    "sepal_length",
-                                                    "petal_length"]
-            num_clusters (int): number of clusters (greater than zero)
-            id_col (string): name of the input id_column
-        """
-        import json
-        from sklearn import metrics
-
-        out_id_colname = 'rowids'
-        # TODO: need a random seed?
-
-        full_query = '''
-            SELECT {cols}, array_agg({id_col}) As {out_id_colname}
-            FROM ({query}) As a
-        '''.format(query=query,
-                   id_col=id_col,
-                   out_id_colname=out_id_colname,
-                   cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
-                                   for idx, val in enumerate(colnames)]))
-
-        db_resp = self.query_runner.get_nonspatial_kmeans(full_query, standarize)
-
-        # fill array with values for k-means clustering
-        if standarize:
-            cluster_columns = _scale_data(
-              _extract_columns(db_resp, colnames))
-        else:
-            cluster_columns = _extract_columns(db_resp, colnames)
-
-        print str(cluster_columns)
-        # TODO: decide on optimal parameters for most cases
-        #       Are there ways of deciding parameters based on inputs?
-        kmeans = KMeans(n_clusters=num_clusters,
-                        random_state=0).fit(cluster_columns)
-
-        centers = [json.dumps(dict(zip(colnames, c)))
-                   for c in kmeans.cluster_centers_[kmeans.labels_]]
-
-        silhouettes = metrics.silhouette_samples(cluster_columns,
-                                                 kmeans.labels_,
-                                                 metric='sqeuclidean')
-
-        return zip(kmeans.labels_,
-                   centers,
-                   silhouettes,
-                   db_resp[0][out_id_colname])
-
-
-# -- Preprocessing steps
-
-def _extract_columns(db_resp, colnames):
-    """
-        Extract the features from the query and pack them into a NumPy array
-        db_resp (plpy data object): result of the kmeans request
-        id_col_name (string): name of column which has the row id (not a
-                              feature of the analysis)
-    """
-    return np.array([db_resp[0][c] for c in colnames],
-                    dtype=float).T
-
-
-def _scale_data(features):
-    """
-        Scale all input columns to center on 0 with a standard devation of 1
-
-        features (numpy matrix): features of dimension (n_features, n_samples)
-    """
-    from sklearn.preprocessing import StandardScaler
-    return StandardScaler().fit_transform(features)
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 9fd2504..879dab7 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -54,36 +54,3 @@ class KMeansTest(unittest.TestCase):
         self.assertEqual(len(np.unique(labels)), 2)
         self.assertEqual(len(c1), 20)
         self.assertEqual(len(c2), 20)
-
-
-class KMeansNonspatialTest(unittest.TestCase):
-    """Testing class for k-means non-spatial"""
-
-    def setUp(self):
-        self.params = {"subquery": "SELECT * FROM TABLE",
-                       "n_clusters": 5}
-
-    def test_kmeans_nonspatial(self):
-        """
-            test for k-means non-spatial
-        """
-        # data from:
-        # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans
-        data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
-                                 ("col2", [2, 4, 0, 2, 4, 0]),
-                                 ("rowids", [1, 2, 3, 4, 5, 6])])]
-
-        random_seeds.set_random_seeds(1234)
-        kmeans = Kmeans(FakeQueryRunner(data_raw))
-        print 'asfasdfasd'
-        clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2)
-        print str([c[0] for c in clusters])
-
-        cl1 = clusters[0][0]
-        cl2 = clusters[3][0]
-
-        for idx, val in enumerate(clusters):
-            if idx < 3:
-                self.assertEqual(val[0], cl1)
-            else:
-                self.assertEqual(val[0], cl2)

From bb3ff43f0fd4d898c6bee9b16d8cf78c8c3ec655 Mon Sep 17 00:00:00 2001
From: Mario de Frutos <ethervoid@gmail.com>
Date: Mon, 21 Nov 2016 17:25:08 +0100
Subject: [PATCH 35/40] Update .travis.yml

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a5f544a..2e7ef27 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,8 +39,8 @@ before_install:
   - sudo apt-get -y autoremove
 
   - sudo apt-get -y install postgresql-9.5=9.5.2-3cdb1
-  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-2ubuntu1
-  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-2ubuntu1
+  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb1
+  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb1
   - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
   - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
 

From 280a5193efc06ad5720c735380e1247bc8e63c3b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 22 Nov 2016 09:32:39 -0500
Subject: [PATCH 36/40] rename queryrunner to analysisdataprovider

---
 src/py/crankshaft/crankshaft/__init__.py      |  2 +-
 ...ry_runner.py => analysis_data_provider.py} |  2 +-
 .../crankshaft/clustering/kmeans.py           | 12 +++++------
 .../crankshaft/crankshaft/clustering/moran.py | 20 +++++++++----------
 .../crankshaft/space_time_dynamics/markov.py  | 12 +++++------
 5 files changed, 24 insertions(+), 24 deletions(-)
 rename src/py/crankshaft/crankshaft/{query_runner.py => analysis_data_provider.py} (97%)

diff --git a/src/py/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py
index a03b040..a8060f8 100644
--- a/src/py/crankshaft/crankshaft/__init__.py
+++ b/src/py/crankshaft/crankshaft/__init__.py
@@ -3,4 +3,4 @@ import crankshaft.random_seeds
 import crankshaft.clustering
 import crankshaft.space_time_dynamics
 import crankshaft.segmentation
-import query_runner
+import analysis_data_provider
diff --git a/src/py/crankshaft/crankshaft/query_runner.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
similarity index 97%
rename from src/py/crankshaft/crankshaft/query_runner.py
rename to src/py/crankshaft/crankshaft/analysis_data_provider.py
index 5775e72..ad572ec 100644
--- a/src/py/crankshaft/crankshaft/query_runner.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -2,7 +2,7 @@
 import plpy
 
 
-class QueryRunner:
+class AnalysisDataProvider:
     def get_markov(self, query):
         try:
             data = plpy.execute(query)
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 200dc52..e59c25b 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -1,15 +1,15 @@
 from sklearn.cluster import KMeans
 import numpy as np
 
-from crankshaft.query_runner import QueryRunner
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 
 
 class Kmeans:
-    def __init__(self, query_runner=None):
-        if query_runner is None:
-            self.query_runner = QueryRunner()
+    def __init__(self, data_provider=None):
+        if data_provider is None:
+            self.data_provider = AnalysisDataProvider()
         else:
-            self.query_runner = query_runner
+            self.data_provider = data_provider
 
     def spatial(self, query, no_clusters, no_init=20):
         """
@@ -23,7 +23,7 @@ class Kmeans:
                       "FROM ({query}) As a "
                       "WHERE the_geom IS NOT NULL").format(query=query)
 
-        data = self.query_runner.get_spatial_kmeans(full_query)
+        data = self.data_provider.get_spatial_kmeans(full_query)
 
         # Unpack query response
         xs = data[0]['xs']
diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py
index d2c99d6..7cc9ba5 100644
--- a/src/py/crankshaft/crankshaft/clustering/moran.py
+++ b/src/py/crankshaft/crankshaft/clustering/moran.py
@@ -8,7 +8,7 @@ Moran's I geostatistics (global clustering & outliers presence)
 import pysal as ps
 import plpy
 from collections import OrderedDict
-from crankshaft.query_runner import QueryRunner
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 
 # crankshaft module
 import crankshaft.pysal_utils as pu
@@ -17,11 +17,11 @@ import crankshaft.pysal_utils as pu
 
 
 class Moran:
-    def __init__(self, query_runner=None):
-        if query_runner is None:
-            self.query_runner = QueryRunner()
+    def __init__(self, data_provider=None):
+        if data_provider is None:
+            self.data_provider = AnalysisDataProvider()
         else:
-            self.query_runner = query_runner
+            self.data_provider = data_provider
 
     def global_stat(self, subquery, attr_name,
                     w_type, num_ngbrs, permutations, geom_col, id_col):
@@ -39,7 +39,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_moran(query)
+        result = self.data_provider.get_moran(query)
 
         # collect attributes
         attr_vals = pu.get_attributes(result)
@@ -71,7 +71,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_moran(query)
+        result = self.data_provider.get_moran(query)
 
         attr_vals = pu.get_attributes(result)
         weight = pu.get_weight(result, w_type, num_ngbrs)
@@ -100,7 +100,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_moran(query)
+        result = self.data_provider.get_moran(query)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -132,7 +132,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_moran(query)
+        result = self.data_provider.get_moran(query)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -165,7 +165,7 @@ class Moran:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        result = self.query_runner.get_moran(query)
+        result = self.data_provider.get_moran(query)
 
         # collect attributes
         attr1_vals = pu.get_attributes(result, 1)
diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
index ea8dd32..51db0ef 100644
--- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
@@ -7,15 +7,15 @@ import numpy as np
 import pysal as ps
 import plpy
 import crankshaft.pysal_utils as pu
-from crankshaft.query_runner import QueryRunner
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 
 
 class Markov:
-    def __init__(self, query_runner=None):
-        if query_runner is None:
-            self.query_runner = QueryRunner()
+    def __init__(self, data_provider=None):
+        if data_provider is None:
+            self.data_provider = AnalysisDataProvider()
         else:
-            self.query_runner = query_runner
+            self.data_provider = data_provider
 
     def spatial_trend(self, subquery, time_cols, num_classes=7,
                       w_type='knn', num_ngbrs=5, permutations=0,
@@ -62,7 +62,7 @@ class Markov:
 
         query = pu.construct_neighbor_query(w_type, qvals)
 
-        query_result = self.query_runner.get_markov(query)
+        query_result = self.data_provider.get_markov(query)
 
         # build weight
         weights = pu.get_weight(query_result, w_type)

From 6fe4fc96689934690ed30a241298293f5a39c14b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 22 Nov 2016 09:57:47 -0500
Subject: [PATCH 37/40] rename queryrunner in tests

---
 src/py/crankshaft/test/test_clustering_kmeans.py   |  8 ++++----
 src/py/crankshaft/test/test_clustering_moran.py    | 12 ++++++------
 src/py/crankshaft/test/test_space_time_dynamics.py |  6 +++---
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 879dab7..04f99f6 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -9,7 +9,7 @@ import numpy as np
 # sys.modules['plpy'] = plpy
 from helper import fixture_file
 from crankshaft.clustering import Kmeans
-from crankshaft.query_runner import QueryRunner
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 import crankshaft.clustering as cc
 
 from crankshaft import random_seeds
@@ -17,11 +17,11 @@ import json
 from collections import OrderedDict
 
 
-class FakeQueryRunner(QueryRunner):
+class FakeDataProvider(AnalysisDataProvider):
     def __init__(self, mocked_result):
         self.mocked_result = mocked_result
 
-    def get_spatial_kmeans(self, query):
+    def get_spatial_kmeans(self, w_type, params):
         return self.mocked_result
 
     def get_nonspatial_kmeans(self, query, standarize):
@@ -45,7 +45,7 @@ class KMeansTest(unittest.TestCase):
                  'ids': d['ids']} for d in self.cluster_data]
 
         random_seeds.set_random_seeds(1234)
-        kmeans = Kmeans(FakeQueryRunner(data))
+        kmeans = Kmeans(FakeDataProvider(data))
         clusters = kmeans.spatial('subquery', 2)
         labels = [a[1] for a in clusters]
         c1 = [a for a in clusters if a[1] == 0]
diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py
index 37cf7d0..5c8c5c9 100644
--- a/src/py/crankshaft/test/test_clustering_moran.py
+++ b/src/py/crankshaft/test/test_clustering_moran.py
@@ -3,18 +3,18 @@ import numpy as np
 
 from helper import fixture_file
 from crankshaft.clustering import Moran
-from crankshaft.clustering import QueryRunner
+from crankshaft.clustering import AnalysisDataProvider
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
 from collections import OrderedDict
 
 
-class FakeQueryRunner(QueryRunner):
+class FakeDataProvider(AnalysisDataProvider):
     def __init__(self, mock_data):
         self.mock_result = mock_data
 
-    def get_moran(self, query):
+    def get_moran(self, w_type, params):
         return self.mock_result
 
 
@@ -67,7 +67,7 @@ class MoranTest(unittest.TestCase):
                              ('neighbors', d['neighbors'])])
                 for d in self.neighbors_data]
 
-        moran = Moran(FakeQueryRunner(data))
+        moran = Moran(FakeDataProvider(data))
         random_seeds.set_random_seeds(1234)
         result = moran.local_stat('subquery', 'value',
                                   'knn', 5, 99, 'the_geom', 'cartodb_id')
@@ -86,7 +86,7 @@ class MoranTest(unittest.TestCase):
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
 
         random_seeds.set_random_seeds(1234)
-        moran = Moran(FakeQueryRunner(data))
+        moran = Moran(FakeDataProvider(data))
         result = moran.local_rate_stat('subquery', 'numerator', 'denominator',
                                        'knn', 5, 99, 'the_geom', 'cartodb_id')
         result = [(row[0], row[1]) for row in result]
@@ -102,7 +102,7 @@ class MoranTest(unittest.TestCase):
                  'attr1': d['value'],
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
         random_seeds.set_random_seeds(1235)
-        moran = Moran(FakeQueryRunner(data))
+        moran = Moran(FakeDataProvider(data))
         result = moran.global_stat('table', 'value',
                                    'knn', 5, 99, 'the_geom',
                                    'cartodb_id')
diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py
index e58c7d4..20e659c 100644
--- a/src/py/crankshaft/test/test_space_time_dynamics.py
+++ b/src/py/crankshaft/test/test_space_time_dynamics.py
@@ -9,11 +9,11 @@ from helper import fixture_file
 from crankshaft.space_time_dynamics import Markov
 import crankshaft.space_time_dynamics as std
 from crankshaft import random_seeds
-from crankshaft.query_runner import QueryRunner
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 import json
 
 
-class FakeQueryRunner(QueryRunner):
+class FakeDataProvider(AnalysisDataProvider):
     def __init__(self, data):
         self.mock_result = data
 
@@ -85,7 +85,7 @@ class SpaceTimeTests(unittest.TestCase):
                  'attr15': d['y2009'],
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
         # print(str(data[0]))
-        markov = Markov(FakeQueryRunner(data))
+        markov = Markov(FakeDataProvider(data))
         random_seeds.set_random_seeds(1234)
 
         result = markov.spatial_trend('subquery',

From db501a2f025534550e8e113e5cba8340d7efac36 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 22 Nov 2016 15:20:14 +0000
Subject: [PATCH 38/40] move query generation to inside analysis data provider
 class

---
 .../crankshaft/analysis_data_provider.py      |  8 +-
 .../crankshaft/crankshaft/clustering/moran.py | 76 ++++++++-----------
 .../crankshaft/space_time_dynamics/markov.py  | 14 ++--
 .../crankshaft/test/test_clustering_kmeans.py |  2 +-
 .../test/test_space_time_dynamics.py          |  2 +-
 5 files changed, 47 insertions(+), 55 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index ad572ec..6af37f5 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -1,10 +1,12 @@
 """class for fetching data"""
 import plpy
+import pysal_utils as pu
 
 
 class AnalysisDataProvider:
-    def get_markov(self, query):
+    def get_markov(self, w_type, params):
         try:
+            query = pu.construct_neighbor_query(w_type, params)
             data = plpy.execute(query)
 
             if len(data) == 0:
@@ -14,10 +16,12 @@ class AnalysisDataProvider:
         except plpy.SPIError, err:
             plpy.error('Analysis failed: %s' % err)
 
-    def get_moran(self, query):
+    def get_moran(self, w_type, params):
         """fetch data for moran's i analyses"""
         try:
+            query = pu.construct_neighbor_query(w_type, params)
             data = plpy.execute(query)
+
             # if there are no neighbors, exit
             if len(data) == 0:
                 return pu.empty_zipped_array(2)
diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py
index 7cc9ba5..70a8501 100644
--- a/src/py/crankshaft/crankshaft/clustering/moran.py
+++ b/src/py/crankshaft/crankshaft/clustering/moran.py
@@ -31,15 +31,13 @@ class Moran:
          core clusters with PySAL.
         Andy Eschbacher
         """
-        qvals = OrderedDict([("id_col", id_col),
-                             ("attr1", attr_name),
-                             ("geom_col", geom_col),
-                             ("subquery", subquery),
-                             ("num_ngbrs", num_ngbrs)])
+        params = OrderedDict([("id_col", id_col),
+                              ("attr1", attr_name),
+                              ("geom_col", geom_col),
+                              ("subquery", subquery),
+                              ("num_ngbrs", num_ngbrs)])
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        result = self.data_provider.get_moran(query)
+        result = self.data_provider.get_moran(w_type, params)
 
         # collect attributes
         attr_vals = pu.get_attributes(result)
@@ -63,15 +61,13 @@ class Moran:
         # geometries with attributes that are null are ignored
         # resulting in a collection of not as near neighbors
 
-        qvals = OrderedDict([("id_col", id_col),
-                             ("attr1", attr),
-                             ("geom_col", geom_col),
-                             ("subquery", subquery),
-                             ("num_ngbrs", num_ngbrs)])
+        params = OrderedDict([("id_col", id_col),
+                              ("attr1", attr),
+                              ("geom_col", geom_col),
+                              ("subquery", subquery),
+                              ("num_ngbrs", num_ngbrs)])
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        result = self.data_provider.get_moran(query)
+        result = self.data_provider.get_moran(w_type, params)
 
         attr_vals = pu.get_attributes(result)
         weight = pu.get_weight(result, w_type, num_ngbrs)
@@ -91,16 +87,14 @@ class Moran:
         Moran's I Rate (global)
         Andy Eschbacher
         """
-        qvals = OrderedDict([("id_col", id_col),
-                             ("attr1", numerator),
-                             ("attr2", denominator)
-                             ("geom_col", geom_col),
-                             ("subquery", subquery),
-                             ("num_ngbrs", num_ngbrs)])
+        params = OrderedDict([("id_col", id_col),
+                              ("attr1", numerator),
+                              ("attr2", denominator)
+                              ("geom_col", geom_col),
+                              ("subquery", subquery),
+                              ("num_ngbrs", num_ngbrs)])
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        result = self.data_provider.get_moran(query)
+        result = self.data_provider.get_moran(w_type, params)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -123,16 +117,14 @@ class Moran:
         # geometries with values that are null are ignored
         # resulting in a collection of not as near neighbors
 
-        qvals = OrderedDict([("id_col", id_col),
-                             ("numerator", numerator),
-                             ("denominator", denominator),
-                             ("geom_col", geom_col),
-                             ("subquery", subquery),
-                             ("num_ngbrs", num_ngbrs)])
+        params = OrderedDict([("id_col", id_col),
+                              ("numerator", numerator),
+                              ("denominator", denominator),
+                              ("geom_col", geom_col),
+                              ("subquery", subquery),
+                              ("num_ngbrs", num_ngbrs)])
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        result = self.data_provider.get_moran(query)
+        result = self.data_provider.get_moran(w_type, params)
 
         # collect attributes
         numer = pu.get_attributes(result, 1)
@@ -156,16 +148,14 @@ class Moran:
             Moran's I (local) Bivariate (untested)
         """
 
-        qvals = OrderedDict([("id_col", id_col),
-                             ("attr1", attr1),
-                             ("attr2", attr2),
-                             ("geom_col", geom_col),
-                             ("subquery", subquery),
-                             ("num_ngbrs", num_ngbrs)])
+        params = OrderedDict([("id_col", id_col),
+                              ("attr1", attr1),
+                              ("attr2", attr2),
+                              ("geom_col", geom_col),
+                              ("subquery", subquery),
+                              ("num_ngbrs", num_ngbrs)])
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        result = self.data_provider.get_moran(query)
+        result = self.data_provider.get_moran(w_type, params)
 
         # collect attributes
         attr1_vals = pu.get_attributes(result, 1)
diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
index 51db0ef..a1f0edb 100644
--- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
@@ -54,15 +54,13 @@ class Markov:
         if len(time_cols) < 2:
             plpy.error('More than one time column needs to be passed')
 
-        qvals = {"id_col": id_col,
-                 "time_cols": time_cols,
-                 "geom_col": geom_col,
-                 "subquery": subquery,
-                 "num_ngbrs": num_ngbrs}
+        params = {"id_col": id_col,
+                  "time_cols": time_cols,
+                  "geom_col": geom_col,
+                  "subquery": subquery,
+                  "num_ngbrs": num_ngbrs}
 
-        query = pu.construct_neighbor_query(w_type, qvals)
-
-        query_result = self.data_provider.get_markov(query)
+        query_result = self.data_provider.get_markov(w_type, params)
 
         # build weight
         weights = pu.get_weight(query_result, w_type)
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 04f99f6..93633b0 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -21,7 +21,7 @@ class FakeDataProvider(AnalysisDataProvider):
     def __init__(self, mocked_result):
         self.mocked_result = mocked_result
 
-    def get_spatial_kmeans(self, w_type, params):
+    def get_spatial_kmeans(self, query):
         return self.mocked_result
 
     def get_nonspatial_kmeans(self, query, standarize):
diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py
index 20e659c..d14563e 100644
--- a/src/py/crankshaft/test/test_space_time_dynamics.py
+++ b/src/py/crankshaft/test/test_space_time_dynamics.py
@@ -17,7 +17,7 @@ class FakeDataProvider(AnalysisDataProvider):
     def __init__(self, data):
         self.mock_result = data
 
-    def get_markov(self, query):
+    def get_markov(self, w_type, params):
         return self.mock_result
 
 

From 6ab1c285d946ed14835972fc2456756b8c16234c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Wed, 30 Nov 2016 10:08:36 -0500
Subject: [PATCH 39/40] places query gen in kmeans data provider

---
 .../crankshaft/crankshaft/analysis_data_provider.py   |  8 +++++++-
 src/py/crankshaft/crankshaft/clustering/kmeans.py     | 11 ++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index 6af37f5..1d1cf2b 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -38,8 +38,14 @@ class AnalysisDataProvider:
         except plpy.SPIError, err:
             plpy.error('Analysis failed: %s' % err)
 
-    def get_spatial_kmeans(self, query):
+    def get_spatial_kmeans(self, params):
         """fetch data for spatial kmeans"""
+        query = ("SELECT "
+                 "array_agg({id_col} ORDER BY {id_col}) as ids,"
+                 "array_agg(ST_X({geom_col}) ORDER BY {id_col}) As xs,"
+                 "array_agg(ST_Y({geom_col}) ORDER BY {id_col}) As ys "
+                 "FROM ({subquery}) As a "
+                 "WHERE {geom_col} IS NOT NULL").format(**params)
         try:
             data = plpy.execute(query)
             return data
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index e59c25b..1e49115 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -16,14 +16,11 @@ class Kmeans:
             find centers based on clusters of latitude/longitude pairs
             query: SQL query that has a WGS84 geometry (the_geom)
         """
-        full_query = ("SELECT "
-                      "array_agg(cartodb_id ORDER BY cartodb_id) as ids,"
-                      "array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,"
-                      "array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys "
-                      "FROM ({query}) As a "
-                      "WHERE the_geom IS NOT NULL").format(query=query)
+        params = {"subquery": query,
+                  "geom_col": "the_geom",
+                  "id_col": "cartodb_id"}
 
-        data = self.data_provider.get_spatial_kmeans(full_query)
+        data = self.data_provider.get_spatial_kmeans(params)
 
         # Unpack query response
         xs = data[0]['xs']

From 59dc9434f710f432c391de79281e2bc4c1d9c2e4 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Thu, 1 Dec 2016 17:06:21 -0500
Subject: [PATCH 40/40] moves getis to class-based framework

---
 src/pg/sql/16_getis.sql                       |  5 +-
 .../crankshaft/analysis_data_provider.py      | 16 ++++-
 .../crankshaft/crankshaft/clustering/getis.py | 64 +++++++++----------
 .../crankshaft/crankshaft/clustering/moran.py |  1 -
 .../crankshaft/space_time_dynamics/markov.py  |  1 +
 .../crankshaft/test/test_clustering_getis.py  | 30 +++++----
 .../crankshaft/test/test_clustering_moran.py  |  2 +-
 7 files changed, 68 insertions(+), 51 deletions(-)

diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql
index 578f15a..e520227 100644
--- a/src/pg/sql/16_getis.sql
+++ b/src/pg/sql/16_getis.sql
@@ -11,8 +11,9 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE (z_score NUMERIC, p_value NUMERIC, p_z_sim NUMERIC, rowid BIGINT)
 AS $$
-  from crankshaft.clustering import getis_ord
-  return getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
+  from crankshaft.clustering import Getis
+  getis = Getis()
+  return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
 $$ LANGUAGE plpythonu;
 
 -- TODO: make a version that accepts the values as arrays
diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index 1d1cf2b..cbc27bc 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -4,7 +4,21 @@ import pysal_utils as pu
 
 
 class AnalysisDataProvider:
+    def get_getis(self, w_type, params):
+        """fetch data for getis ord's g"""
+        try:
+            query = pu.construct_neighbor_query(w_type, params)
+            result = plpy.execute(query)
+            # if there are no neighbors, exit
+            if len(result) == 0:
+                return pu.empty_zipped_array(4)
+            else:
+                return result
+        except plpy.SPIError, err:
+            plpy.error('Analysis failed: %s' % err)
+
     def get_markov(self, w_type, params):
+        """fetch data for spatial markov"""
         try:
             query = pu.construct_neighbor_query(w_type, params)
             data = plpy.execute(query)
@@ -50,4 +64,4 @@ class AnalysisDataProvider:
             data = plpy.execute(query)
             return data
         except plpy.SPIError, err:
-            plpy.error("Analysis failed: %s" % err)
+            plpy.error('Analysis failed: %s' % err)
diff --git a/src/py/crankshaft/crankshaft/clustering/getis.py b/src/py/crankshaft/crankshaft/clustering/getis.py
index a593e64..bef8f50 100644
--- a/src/py/crankshaft/crankshaft/clustering/getis.py
+++ b/src/py/crankshaft/crankshaft/clustering/getis.py
@@ -3,50 +3,48 @@ Getis-Ord's G geostatistics (hotspot/coldspot analysis)
 """
 
 import pysal as ps
-import plpy
 from collections import OrderedDict
 
-# crankshaft module
+# crankshaft modules
 import crankshaft.pysal_utils as pu
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 
 # High level interface ---------------------------------------
 
 
-def getis_ord(subquery, attr,
-              w_type, num_ngbrs, permutations, geom_col, id_col):
-    """
-    Getis-Ord's G*
-    Implementation building neighbors with a PostGIS database and PySAL's
-      Getis-Ord's G* hotspot/coldspot module.
-    Andy Eschbacher
-    """
+class Getis:
+    def __init__(self, data_provider=None):
+        if data_provider is None:
+            self.data_provider = AnalysisDataProvider()
+        else:
+            self.data_provider = data_provider
 
-    # geometries with attributes that are null are ignored
-    # resulting in a collection of not as near neighbors if kNN is chosen
+    def getis_ord(self, subquery, attr,
+                  w_type, num_ngbrs, permutations, geom_col, id_col):
+        """
+        Getis-Ord's G*
+        Implementation building neighbors with a PostGIS database and PySAL's
+          Getis-Ord's G* hotspot/coldspot module.
+        Andy Eschbacher
+        """
 
-    qvals = OrderedDict([("id_col", id_col),
-                         ("attr1", attr),
-                         ("geom_col", geom_col),
-                         ("subquery", subquery),
-                         ("num_ngbrs", num_ngbrs)])
+        # geometries with attributes that are null are ignored
+        # resulting in a collection of not as near neighbors if kNN is chosen
 
-    query = pu.construct_neighbor_query(w_type, qvals)
+        qvals = OrderedDict([("id_col", id_col),
+                             ("attr1", attr),
+                             ("geom_col", geom_col),
+                             ("subquery", subquery),
+                             ("num_ngbrs", num_ngbrs)])
 
-    try:
-        result = plpy.execute(query)
-        # if there are no neighbors, exit
-        if len(result) == 0:
-            return pu.empty_zipped_array(4)
-    except plpy.SPIError, err:
-        plpy.error('Query failed: %s' % err)
+        result = self.data_provider.get_getis(w_type, qvals)
+        attr_vals = pu.get_attributes(result)
 
-    attr_vals = pu.get_attributes(result)
+        # build PySAL weight object
+        weight = pu.get_weight(result, w_type, num_ngbrs)
 
-    # build PySAL weight object
-    weight = pu.get_weight(result, w_type, num_ngbrs)
+        # calculate Getis-Ord's G* z- and p-values
+        getis = ps.esda.getisord.G_Local(attr_vals, weight,
+                                         star=True, permutations=permutations)
 
-    # calculate Getis-Ord's G* z- and p-values
-    getis = ps.esda.getisord.G_Local(attr_vals, weight,
-                                     star=True, permutations=permutations)
-
-    return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order)
+        return zip(getis.z_sim, getis.p_sim, getis.p_z_sim, weight.id_order)
diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py
index 70a8501..a42a981 100644
--- a/src/py/crankshaft/crankshaft/clustering/moran.py
+++ b/src/py/crankshaft/crankshaft/clustering/moran.py
@@ -6,7 +6,6 @@ Moran's I geostatistics (global clustering & outliers presence)
 #       average of the their neighborhood
 
 import pysal as ps
-import plpy
 from collections import OrderedDict
 from crankshaft.analysis_data_provider import AnalysisDataProvider
 
diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
index f0c0b4a..3ad8273 100644
--- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
+++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py
@@ -2,6 +2,7 @@
 Spatial dynamics measurements using Spatial Markov
 """
 
+# TODO: remove all plpy dependencies
 
 import numpy as np
 import pysal as ps
diff --git a/src/py/crankshaft/test/test_clustering_getis.py b/src/py/crankshaft/test/test_clustering_getis.py
index 835a121..61add11 100644
--- a/src/py/crankshaft/test/test_clustering_getis.py
+++ b/src/py/crankshaft/test/test_clustering_getis.py
@@ -1,18 +1,13 @@
 import unittest
 import numpy as np
 
+from helper import fixture_file
 
-# from mock_plpy import MockPlPy
-# plpy = MockPlPy()
-#
-# import sys
-# sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
-
-import crankshaft.clustering as cc
+from crankshaft.clustering import Getis
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 
 # Fixture files produced as follows
 #
@@ -42,6 +37,14 @@ import json
 #                     lgstar_queen.p_sim, lgstar_queen.p_z_sim)))
 
 
+class FakeDataProvider(AnalysisDataProvider):
+    def __init__(self, mock_data):
+        self.mock_result = mock_data
+
+    def get_getis(self, w_type, param):
+        return self.mock_result
+
+
 class GetisTest(unittest.TestCase):
     """Testing class for Getis-Ord's G* funtion
        This test replicates the work done in PySAL documentation:
@@ -49,8 +52,6 @@ class GetisTest(unittest.TestCase):
     """
 
     def setUp(self):
-        plpy._reset()
-
         # load raw data for analysis
         self.neighbors_data = json.loads(
           open(fixture_file('neighbors_getis.json')).read())
@@ -64,10 +65,13 @@ class GetisTest(unittest.TestCase):
         data = [{'id': d['id'],
                  'attr1': d['value'],
                  'neighbors': d['neighbors']} for d in self.neighbors_data]
-        plpy._define_result('select', data)
+
         random_seeds.set_random_seeds(1234)
-        result = cc.getis_ord('subquery', 'value',
-                              'queen', None, 999, 'the_geom', 'cartodb_id')
+        getis = Getis(FakeDataProvider(data))
+
+        result = getis.getis_ord('subquery', 'value',
+                                 'queen', None, 999, 'the_geom',
+                                 'cartodb_id')
         result = [(row[0], row[1]) for row in result]
         expected = np.array(self.getis_data)[:, 0:2]
         for ([res_z, res_p], [exp_z, exp_p]) in zip(result, expected):
diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py
index 5c8c5c9..cc1930e 100644
--- a/src/py/crankshaft/test/test_clustering_moran.py
+++ b/src/py/crankshaft/test/test_clustering_moran.py
@@ -3,7 +3,7 @@ import numpy as np
 
 from helper import fixture_file
 from crankshaft.clustering import Moran
-from crankshaft.clustering import AnalysisDataProvider
+from crankshaft.analysis_data_provider import AnalysisDataProvider
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json