From 798e754dfb091789e84ca03476acd0813785dd5e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Mon, 5 Dec 2016 17:14:36 -0500
Subject: [PATCH 01/24] stubs in kmeans non-spatial

---
 .../crankshaft/analysis_data_provider.py      |  9 ++-
 .../crankshaft/clustering/kmeans.py           | 72 +++++++++++++++++++
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index cbc27bc..34512ae 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -44,8 +44,15 @@ class AnalysisDataProvider:
             plpy.error('Analysis failed: %s' % e)
             return pu.empty_zipped_array(2)
 
-    def get_nonspatial_kmeans(self, query):
+    def get_nonspatial_kmeans(self, params):
         """fetch data for non-spatial kmeans"""
+        query = '''
+            SELECT {cols}, array_agg({id_col}) As rowid
+            FROM ({subquery}) As a
+        '''.format(subquery=subquery,
+                   id_col=id_col,
+                   cols=', '.join(['array_agg({0}) As arr_{0}'.format(c)
+                                   for c in params[colnames]]))
         try:
             data = plpy.execute(query)
             return data
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 1e49115..84d3e0a 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -30,3 +30,75 @@ class Kmeans:
         km = KMeans(n_clusters=no_clusters, n_init=no_init)
         labels = km.fit_predict(zip(xs, ys))
         return zip(ids, labels)
+
+    def nonspatial(self, subquery, colnames, num_clusters=5,
+                   id_col='cartodb_id', standarize=True):
+        """
+            query (string): A SQL query to retrieve the data required to do the
+                            k-means clustering analysis, like so:
+                            SELECT * FROM iris_flower_data
+            colnames (list): a list of the column names which contain the data
+                             of interest, like so: ["sepal_width",
+                                                    "petal_width",
+                                                    "sepal_length",
+                                                    "petal_length"]
+            num_clusters (int): number of clusters (greater than zero)
+            id_col (string): name of the input id_column
+        """
+        import json
+        from sklearn import metrics
+
+        out_id_colname = 'rowids'
+        # TODO: need a random seed?
+        params = {"cols": colnames,
+                  "subquery": subquery,
+                  "id_col": id_col}
+
+        data = self.query_runner.get_nonspatial_kmeans(params, standarize)
+
+        # fill array with values for k-means clustering
+        if standarize:
+            cluster_columns = _scale_data(
+              _extract_columns(data, colnames))
+        else:
+            cluster_columns = _extract_columns(data, colnames)
+
+        print str(cluster_columns)
+        # TODO: decide on optimal parameters for most cases
+        #       Are there ways of deciding parameters based on inputs?
+        kmeans = KMeans(n_clusters=num_clusters,
+                        random_state=0).fit(cluster_columns)
+
+        centers = [json.dumps(dict(zip(colnames, c)))
+                   for c in kmeans.cluster_centers_[kmeans.labels_]]
+
+        silhouettes = metrics.silhouette_samples(cluster_columns,
+                                                 kmeans.labels_,
+                                                 metric='sqeuclidean')
+
+        return zip(kmeans.labels_,
+                   centers,
+                   silhouettes,
+                   data[0][out_id_colname])
+
+
+# -- Preprocessing steps
+
+def _extract_columns(data, colnames):
+    """
+        Extract the features from the query and pack them into a NumPy array
+        data (list of dicts): result of the kmeans request
+        id_col_name (string): name of column which has the row id (not a
+                              feature of the analysis)
+    """
+    return np.array([data[0]['arr_{}'.format(c)] for c in colnames],
+                    dtype=float).T
+
+
+def _scale_data(features):
+    """
+        Scale all input columns to center on 0 with a standard devation of 1
+        features (numpy matrix): features of dimension (n_features, n_samples)
+    """
+    from sklearn.preprocessing import StandardScaler
+    return StandardScaler().fit_transform(features)

From 9a80244e76df5c33034cd0fdc0ce34dade3a193e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 10:14:37 -0500
Subject: [PATCH 02/24] adds tests and pgsql file

---
 src/pg/sql/11_kmeans.sql                      | 19 +++++++++
 .../crankshaft/test/test_clustering_kmeans.py | 39 ++++++++++++++++---
 2 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 0899e81..6100d27 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -9,6 +9,25 @@ RETURNS table (cartodb_id integer, cluster_no integer) as $$
 
 $$ LANGUAGE plpythonu;
 
+-- Non-spatial k-means clustering
+-- query: sql query to retrieve all the needed data
+
+CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
+  query TEXT,
+  colnames TEXT[],
+  num_clusters INTEGER,
+  id_colname TEXT DEFAULT 'cartodb_id',
+  standarize BOOLEAN DEFAULT true
+)
+RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
+
+    from crankshaft.clustering import Kmeans
+    kmeans = Kmeans()
+    return kmeans.nonspatial(query, colnames, num_clusters,
+                             id_colname, standarize)
+$$ LANGUAGE plpythonu;
+
+
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)
 RETURNS Numeric[] AS
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 93633b0..5e096ca 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -2,17 +2,12 @@ import unittest
 import numpy as np
 
 
-# from mock_plpy import MockPlPy
-# plpy = MockPlPy()
-#
-# import sys
-# sys.modules['plpy'] = plpy
 from helper import fixture_file
 from crankshaft.clustering import Kmeans
 from crankshaft.analysis_data_provider import AnalysisDataProvider
 import crankshaft.clustering as cc
-
 from crankshaft import random_seeds
+
 import json
 from collections import OrderedDict
 
@@ -54,3 +49,35 @@ class KMeansTest(unittest.TestCase):
         self.assertEqual(len(np.unique(labels)), 2)
         self.assertEqual(len(c1), 20)
         self.assertEqual(len(c2), 20)
+
+
+class KMeansNonspatialTest(unittest.TestCase):
+    """Testing class for k-means non-spatial"""
+
+    def setUp(self):
+        self.params = {"subquery": "SELECT * FROM TABLE",
+                       "n_clusters": 5}
+
+    def test_kmeans_nonspatial(self):
+        """
+            test for k-means non-spatial
+        """
+        # data from:
+        # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans
+        data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
+                                 ("col2", [2, 4, 0, 2, 4, 0]),
+                                 ("rowids", [1, 2, 3, 4, 5, 6])])]
+
+        random_seeds.set_random_seeds(1234)
+        kmeans = Kmeans(FakeQueryRunner(data_raw))
+        clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2)
+        print str([c[0] for c in clusters])
+
+        cl1 = clusters[0][0]
+        cl2 = clusters[3][0]
+
+        for idx, val in enumerate(clusters):
+            if idx < 3:
+                self.assertEqual(val[0], cl1)
+            else:
+                self.assertEqual(val[0], cl2)

From e98f1cbce5a0d4a01d1823c34ee8e44d628c1c1f Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 10:19:13 -0500
Subject: [PATCH 03/24] fix query formatting with dict

---
 src/py/crankshaft/crankshaft/analysis_data_provider.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index 34512ae..973bd1b 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -49,10 +49,10 @@ class AnalysisDataProvider:
         query = '''
             SELECT {cols}, array_agg({id_col}) As rowid
             FROM ({subquery}) As a
-        '''.format(subquery=subquery,
-                   id_col=id_col,
+        '''.format(subquery=params['subquery'],
+                   id_col=params['id_col'],
                    cols=', '.join(['array_agg({0}) As arr_{0}'.format(c)
-                                   for c in params[colnames]]))
+                                   for c in params['colnames']]))
         try:
             data = plpy.execute(query)
             return data

From b65fa0c6134c6897d84c4e2589b1a05589170418 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 10:27:24 -0500
Subject: [PATCH 04/24] remove erroneous queryrunner class

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 5e096ca..c78861d 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -69,7 +69,7 @@ class KMeansNonspatialTest(unittest.TestCase):
                                  ("rowids", [1, 2, 3, 4, 5, 6])])]
 
         random_seeds.set_random_seeds(1234)
-        kmeans = Kmeans(FakeQueryRunner(data_raw))
+        kmeans = Kmeans(FakeDataProvider(data_raw))
         clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2)
         print str([c[0] for c in clusters])
 

From c884eae90e41577670b8bd194cc55b31e49f3f61 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 10:33:51 -0500
Subject: [PATCH 05/24] fix data provider ref

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 84d3e0a..2f53d15 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -54,7 +54,7 @@ class Kmeans:
                   "subquery": subquery,
                   "id_col": id_col}
 
-        data = self.query_runner.get_nonspatial_kmeans(params, standarize)
+        data = self.data_provider.get_nonspatial_kmeans(params, standarize)
 
         # fill array with values for k-means clustering
         if standarize:

From cc0a683a268a62f197b9fee4ae11c05fb44cea03 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 12:26:25 -0500
Subject: [PATCH 06/24] fix query templating / response access

---
 src/py/crankshaft/crankshaft/analysis_data_provider.py | 6 ++++--
 src/py/crankshaft/crankshaft/clustering/kmeans.py      | 9 +++++----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index 973bd1b..eb730ff 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -46,13 +46,15 @@ class AnalysisDataProvider:
 
     def get_nonspatial_kmeans(self, params):
         """fetch data for non-spatial kmeans"""
+        agg_cols = ', '.join(['array_agg({0}) As arr_col{1}'.format(idx+1, val)
+                              for idx, val in enumerate(params['colnames'])])
+
         query = '''
             SELECT {cols}, array_agg({id_col}) As rowid
             FROM ({subquery}) As a
         '''.format(subquery=params['subquery'],
                    id_col=params['id_col'],
-                   cols=', '.join(['array_agg({0}) As arr_{0}'.format(c)
-                                   for c in params['colnames']]))
+                   cols=agg_cols)
         try:
             data = plpy.execute(query)
             return data
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 2f53d15..bb3343b 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -59,9 +59,9 @@ class Kmeans:
         # fill array with values for k-means clustering
         if standarize:
             cluster_columns = _scale_data(
-              _extract_columns(data, colnames))
+              _extract_columns(data, len(colnames)))
         else:
-            cluster_columns = _extract_columns(data, colnames)
+            cluster_columns = _extract_columns(data, len(colnames))
 
         print str(cluster_columns)
         # TODO: decide on optimal parameters for most cases
@@ -84,14 +84,15 @@ class Kmeans:
 
 # -- Preprocessing steps
 
-def _extract_columns(data, colnames):
+def _extract_columns(data, n_cols):
     """
         Extract the features from the query and pack them into a NumPy array
         data (list of dicts): result of the kmeans request
         id_col_name (string): name of column which has the row id (not a
                               feature of the analysis)
     """
-    return np.array([data[0]['arr_{}'.format(c)] for c in colnames],
+    return np.array([data[0]['arr_col{0}'.format(i+1)]
+                     for i in xrange(n_cols)],
                     dtype=float).T
 
 

From bb5de09d6d06a9ffa2c46f86e00b80e3478f2512 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 12:49:27 -0500
Subject: [PATCH 07/24] update order of query gen

---
 src/py/crankshaft/crankshaft/analysis_data_provider.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index eb730ff..d942627 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -46,9 +46,9 @@ class AnalysisDataProvider:
 
     def get_nonspatial_kmeans(self, params):
         """fetch data for non-spatial kmeans"""
-        agg_cols = ', '.join(['array_agg({0}) As arr_col{1}'.format(idx+1, val)
+        agg_cols = ', '.join(['array_agg({0}) As arr_col{1}'.format(val, idx+1)
                               for idx, val in enumerate(params['colnames'])])
-
+        print agg_cols
         query = '''
             SELECT {cols}, array_agg({id_col}) As rowid
             FROM ({subquery}) As a

From 3dad9c604476a5c415b39cd5d828139a66f05d4e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 6 Dec 2016 13:45:04 -0500
Subject: [PATCH 08/24] update key name in test

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index c78861d..572a514 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -64,8 +64,8 @@ class KMeansNonspatialTest(unittest.TestCase):
         """
         # data from:
         # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans
-        data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
-                                 ("col2", [2, 4, 0, 2, 4, 0]),
+        data_raw = [OrderedDict([("arr_col1", [1, 1, 1, 4, 4, 4]),
+                                 ("arr_col2", [2, 4, 0, 2, 4, 0]),
                                  ("rowids", [1, 2, 3, 4, 5, 6])])]
 
         random_seeds.set_random_seeds(1234)

From c6f64ad2f401a0b2f14a17453dcc9610e40f01c1 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 10 Jan 2017 09:49:16 -0500
Subject: [PATCH 09/24] bug fixes and adding of internal docs

---
 src/pg/sql/11_kmeans.sql                      | 11 ++++--
 .../crankshaft/analysis_data_provider.py      | 25 ++++++++++++--
 .../crankshaft/clustering/kmeans.py           | 34 +++++++++++--------
 .../crankshaft/test/test_clustering_kmeans.py |  4 +--
 4 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 6100d27..89e16a8 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -11,20 +11,25 @@ $$ LANGUAGE plpythonu;
 
 -- Non-spatial k-means clustering
 -- query: sql query to retrieve all the needed data
+-- colnames: text array of column names for doing the clustering analysis
+-- standardize: whether to scale variables to a mean of zero and a standard
+--              deviation of 1
+-- id_colname: name of the id column
 
 CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
   query TEXT,
   colnames TEXT[],
   num_clusters INTEGER,
-  id_colname TEXT DEFAULT 'cartodb_id',
-  standarize BOOLEAN DEFAULT true
+  standardize BOOLEAN DEFAULT true,
+  id_colname TEXT DEFAULT 'cartodb_id'
 )
 RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
 
     from crankshaft.clustering import Kmeans
     kmeans = Kmeans()
     return kmeans.nonspatial(query, colnames, num_clusters,
-                             id_colname, standarize)
+                             standardize=standardize,
+                             id_col=id_colname)
 $$ LANGUAGE plpythonu;
 
 
diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index d942627..ef9baed 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -45,18 +45,34 @@ class AnalysisDataProvider:
             return pu.empty_zipped_array(2)
 
     def get_nonspatial_kmeans(self, params):
-        """fetch data for non-spatial kmeans"""
+        """
+            Fetch data for non-spatial k-means.
+
+            Inputs - a dict (params) with the following keys:
+                colnames: a (text) list of column names (e.g.,
+                          `['andy', 'cookie']`)
+                id_col: the name of the id column (e.g., `'cartodb_id'`)
+                subquery: the subquery for exposing the data (e.g.,
+                          SELECT * FROM favorite_things)
+            Output:
+                A SQL query for packaging the data for consumption within
+                `KMeans().nonspatial`. Format will be a list of length one,
+                with the first element a dict with keys ('rowid', 'attr1',
+                'attr2', ...)
+        """
         agg_cols = ', '.join(['array_agg({0}) As arr_col{1}'.format(val, idx+1)
                               for idx, val in enumerate(params['colnames'])])
-        print agg_cols
         query = '''
             SELECT {cols}, array_agg({id_col}) As rowid
             FROM ({subquery}) As a
         '''.format(subquery=params['subquery'],
                    id_col=params['id_col'],
-                   cols=agg_cols)
+                   cols=agg_cols).strip()
         try:
             data = plpy.execute(query)
+            if len(data) == 0:
+                plpy.error('No non-null-valued data to analyze. Check the '
+                           'rows and columns of all of the inputs')
             return data
         except plpy.SPIError, err:
             plpy.error('Analysis failed: %s' % err)
@@ -71,6 +87,9 @@ class AnalysisDataProvider:
                  "WHERE {geom_col} IS NOT NULL").format(**params)
         try:
             data = plpy.execute(query)
+            if len(data) == 0:
+                plpy.error('No non-null-valued data to analyze. Check the '
+                           'rows and columns of all of the inputs')
             return data
         except plpy.SPIError, err:
             plpy.error('Analysis failed: %s' % err)
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index bb3343b..2477d80 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -32,40 +32,45 @@ class Kmeans:
         return zip(ids, labels)
 
     def nonspatial(self, subquery, colnames, num_clusters=5,
-                   id_col='cartodb_id', standarize=True):
+                   standardize=True, id_col='cartodb_id'):
         """
+            Inputs:
             query (string): A SQL query to retrieve the data required to do the
                             k-means clustering analysis, like so:
                             SELECT * FROM iris_flower_data
             colnames (list): a list of the column names which contain the data
-                             of interest, like so: ["sepal_width",
-                                                    "petal_width",
-                                                    "sepal_length",
-                                                    "petal_length"]
+                             of interest, like so: ['sepal_width',
+                                                    'petal_width',
+                                                    'sepal_length',
+                                                    'petal_length']
             num_clusters (int): number of clusters (greater than zero)
             id_col (string): name of the input id_column
+
+            Output:
+            A list of tuples with the following columns:
+            cluster labels: a label for the cluster that the row belongs to
+            centers: center of the cluster that this row belongs to
+            silhouettes: silhouette measure for this value
+            rowid: row that these values belong to (corresponds to the value in
+                   `id_col`)
         """
         import json
         from sklearn import metrics
 
-        out_id_colname = 'rowids'
         # TODO: need a random seed?
-        params = {"cols": colnames,
+        params = {"colnames": colnames,
                   "subquery": subquery,
                   "id_col": id_col}
 
-        data = self.data_provider.get_nonspatial_kmeans(params, standarize)
+        data = self.data_provider.get_nonspatial_kmeans(params)
 
         # fill array with values for k-means clustering
-        if standarize:
+        if standardize:
             cluster_columns = _scale_data(
               _extract_columns(data, len(colnames)))
         else:
             cluster_columns = _extract_columns(data, len(colnames))
 
-        print str(cluster_columns)
-        # TODO: decide on optimal parameters for most cases
-        #       Are there ways of deciding parameters based on inputs?
         kmeans = KMeans(n_clusters=num_clusters,
                         random_state=0).fit(cluster_columns)
 
@@ -79,7 +84,7 @@ class Kmeans:
         return zip(kmeans.labels_,
                    centers,
                    silhouettes,
-                   data[0][out_id_colname])
+                   data[0]['rowid'])
 
 
 # -- Preprocessing steps
@@ -102,4 +107,5 @@ def _scale_data(features):
         features (numpy matrix): features of dimension (n_features, n_samples)
     """
     from sklearn.preprocessing import StandardScaler
-    return StandardScaler().fit_transform(features)
+    scaler = StandardScaler()
+    return scaler.fit_transform(features)
diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 572a514..3756b7e 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -19,7 +19,7 @@ class FakeDataProvider(AnalysisDataProvider):
     def get_spatial_kmeans(self, query):
         return self.mocked_result
 
-    def get_nonspatial_kmeans(self, query, standarize):
+    def get_nonspatial_kmeans(self, query):
         return self.mocked_result
 
 
@@ -66,7 +66,7 @@ class KMeansNonspatialTest(unittest.TestCase):
         # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans
         data_raw = [OrderedDict([("arr_col1", [1, 1, 1, 4, 4, 4]),
                                  ("arr_col2", [2, 4, 0, 2, 4, 0]),
-                                 ("rowids", [1, 2, 3, 4, 5, 6])])]
+                                 ("rowid", [1, 2, 3, 4, 5, 6])])]
 
         random_seeds.set_random_seeds(1234)
         kmeans = Kmeans(FakeDataProvider(data_raw))

From 69f38dd52e2d625364d2b77969ddeb7d87b3ff1c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 10 Jan 2017 09:52:46 -0500
Subject: [PATCH 10/24] change parameter name to align with kmeans.spatial

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 2477d80..fe6831f 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -31,7 +31,7 @@ class Kmeans:
         labels = km.fit_predict(zip(xs, ys))
         return zip(ids, labels)
 
-    def nonspatial(self, subquery, colnames, num_clusters=5,
+    def nonspatial(self, subquery, colnames, no_clusters=5,
                    standardize=True, id_col='cartodb_id'):
         """
             Inputs:
@@ -43,7 +43,7 @@ class Kmeans:
                                                     'petal_width',
                                                     'sepal_length',
                                                     'petal_length']
-            num_clusters (int): number of clusters (greater than zero)
+            no_clusters (int): number of clusters (greater than zero)
             id_col (string): name of the input id_column
 
             Output:
@@ -71,7 +71,7 @@ class Kmeans:
         else:
             cluster_columns = _extract_columns(data, len(colnames))
 
-        kmeans = KMeans(n_clusters=num_clusters,
+        kmeans = KMeans(n_clusters=no_clusters,
                         random_state=0).fit(cluster_columns)
 
         centers = [json.dumps(dict(zip(colnames, c)))

From a32b212412b3429a121f6e635fc3080787829c9b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 10 Jan 2017 10:43:42 -0500
Subject: [PATCH 11/24] finish docs for kmeans nonspatial

---
 doc/11_kmeans.md | 71 +++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 55 insertions(+), 16 deletions(-)

diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md
index 6153010..93926cf 100644
--- a/doc/11_kmeans.md
+++ b/doc/11_kmeans.md
@@ -2,9 +2,7 @@
 
 ### CDB_KMeans(subquery text, no_clusters INTEGER)
 
-This function attempts to find n clusters within the input data. It will return a table to CartoDB ids and 
-the number of the cluster each point in the input was assigend to.
-
+This function attempts to find `no_clusters` clusters within the input data based on the geographic distribution. It will return a table with ids and the cluster classification of each point input assuming `the_geom` is not null-valued. If `the_geom` is null-valued, the point will not be considered in the analysis.
 
 #### Arguments
 
@@ -19,17 +17,19 @@ A table with the following columns.
 
 | Column Name | Type | Description |
 |-------------|------|-------------|
-| cartodb\_id | INTEGER | The CartoDB id of the row in the input table.|
-| cluster\_no | INTEGER | The cluster that this point belongs to. |
+| cartodb\_id | INTEGER | The row id of the row from the input table |
+| cluster\_no | INTEGER | The cluster that this point belongs to |
 
 
 #### Example Usage
 
 ```sql
-SELECT 
-    customers.*, 
-    km.cluster_no 
-    FROM cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) km, customers_3
+SELECT
+    customers.*,
+    km.cluster_no
+    FROM
+      cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) As km,
+      customers
     WHERE customers.cartodb_id = km.cartodb_id
 ```
 
@@ -37,7 +37,7 @@ SELECT
 
 Function that computes the weighted centroid of a number of clusters by some weight column.
 
-### Arguments 
+### Arguments
 
 | Name | Type | Description |
 |------|------|-------------|
@@ -45,18 +45,57 @@ Function that computes the weighted centroid of a number of clusters by some wei
 | weight\_column | TEXT | The name of the column to use as a weight |
 | category\_column | TEXT | The name of the column to use as a category |
 
-### Returns 
+### Returns
 
 A table with the following columns.
 
 | Column Name | Type | Description |
 |-------------|------|-------------|
 | the\_geom | GEOMETRY | A point for the weighted cluster center |
-| class | INTEGER | The cluster class | 
+| class | INTEGER | The cluster class |
 
-### Example Usage 
+### Example Usage
 
-```sql 
-SELECT ST_TRANSFORM(the_geom, 3857) as the_geom_webmercator, class 
-FROM cdb_weighted_mean('SELECT *, customer_value FROM customers','customer_value','cluster_no')
+```sql
+SELECT
+  ST_Transform(the_geom, 3857) As the_geom_webmercator,
+  class
+FROM
+  cdb_crankshaft.CDB_Weighted_Mean(
+    'SELECT *, customer_value FROM customers',
+    'customer_value',
+    'cluster_no')
 ```
+
+## CDB_KMeansNonspatial(subquery text, colnames text[], no_clusters int)
+
+K-means clustering classifies the rows of your dataset into `no_clusters` by finding the centers (means) of the variables in `colnames` and classifying each row by it's proximity to the nearest center. This method partitions space into distinct Voronoi cells.
+
+As a standard machine learning method, k-means clustering is an unsupervised learning technique that finds the natural clustering of values. For instance, it is useful for finding subgroups in census data leading to demographic segmentation.
+
+### Arguments
+
+| Name | Type | Description |
+|------|------|-------------|
+| query | TEXT | SQL query to expose the data to be used in the analysis (e.g., `SELECT * FROM iris_data`). It should contain at least the columns specified in `colnames` and the `id_colname`. |
+| colnames | TEXT[] | Array of columns to be used in the analysis (e.g., `Array['petal_width', 'sepal_length', 'petal_length']`). |
+| no\_clusters | INTEGER | Number of clusters for the classification of the data |
+| id_colname (optaional) | TEXT | The id column (default: 'cartodb_id') for identifying rows |
+| standarize (optional) | BOOLEAN | Setting this to true (default) standardizes the data to have a mean at zero and a standard deviation of 1 |
+
+### Returns
+
+A table with the following columns.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| cluster_label | TEXT | Label that a cluster belongs to, number from 0 to `no_clusters - 1`. |
+| cluster_center | JSON | Center of the cluster that a row belongs to. The keys of the JSON object are the `colnames`, with values that are the center of the respective cluster |
+| silhouettes | NUMERIC | [Silhouette score](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.silhouette_score.html#sklearn.metrics.silhouette_score) of the cluster label |
+| rowid | BIGINT | id of the original row for associating back with the original data |
+
+
+### Resources
+
+-   Read more in [scikit-learn's documentation](http://scikit-learn.org/stable/modules/clustering.html#k-means)
+-   [K-means basics](https://www.datascience.com/blog/introduction-to-k-means-clustering-algorithm-learn-data-science-tutorials)

From ee5e7d81ae66585b8b27d4f812084afa85e59df5 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 10 Jan 2017 10:52:10 -0500
Subject: [PATCH 12/24] standardize id_col naming convention

---
 doc/11_kmeans.md         | 2 +-
 src/pg/sql/11_kmeans.sql | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md
index 93926cf..4a21faa 100644
--- a/doc/11_kmeans.md
+++ b/doc/11_kmeans.md
@@ -80,7 +80,7 @@ As a standard machine learning method, k-means clustering is an unsupervised lea
 | query | TEXT | SQL query to expose the data to be used in the analysis (e.g., `SELECT * FROM iris_data`). It should contain at least the columns specified in `colnames` and the `id_colname`. |
 | colnames | TEXT[] | Array of columns to be used in the analysis (e.g., `Array['petal_width', 'sepal_length', 'petal_length']`). |
 | no\_clusters | INTEGER | Number of clusters for the classification of the data |
-| id_colname (optaional) | TEXT | The id column (default: 'cartodb_id') for identifying rows |
+| id_col (optional) | TEXT | The id column (default: 'cartodb_id') for identifying rows |
 | standarize (optional) | BOOLEAN | Setting this to true (default) standardizes the data to have a mean at zero and a standard deviation of 1 |
 
 ### Returns
diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 89e16a8..7aad2d8 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -21,7 +21,7 @@ CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
   colnames TEXT[],
   num_clusters INTEGER,
   standardize BOOLEAN DEFAULT true,
-  id_colname TEXT DEFAULT 'cartodb_id'
+  id_col TEXT DEFAULT 'cartodb_id'
 )
 RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
 
@@ -29,7 +29,7 @@ RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowi
     kmeans = Kmeans()
     return kmeans.nonspatial(query, colnames, num_clusters,
                              standardize=standardize,
-                             id_col=id_colname)
+                             id_col=id_col)
 $$ LANGUAGE plpythonu;
 
 

From 06746b4c6596f9e1f321185e211417023000bb0c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <ohasselblad@users.noreply.github.com>
Date: Thu, 19 Jan 2017 08:40:10 -0500
Subject: [PATCH 13/24] corrected code snippet on weighted mean

---
 doc/11_kmeans.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md
index 4a21faa..cd4a2fa 100644
--- a/doc/11_kmeans.md
+++ b/doc/11_kmeans.md
@@ -61,7 +61,7 @@ SELECT
   ST_Transform(the_geom, 3857) As the_geom_webmercator,
   class
 FROM
-  cdb_crankshaft.CDB_Weighted_Mean(
+  cdb_crankshaft.CDB_WeightedMean(
     'SELECT *, customer_value FROM customers',
     'customer_value',
     'cluster_no')

From 5e0fbf0f6fe293761216c7a0bc719b4c9dc4b32b Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 13:02:41 -0500
Subject: [PATCH 14/24] syntax updates

---
 src/pg/sql/11_kmeans.sql | 49 +++++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 16 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 7aad2d8..1c34b75 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -1,17 +1,25 @@
 -- Spatial k-means clustering
 
-CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer, no_init integer default 20)
-RETURNS table (cartodb_id integer, cluster_no integer) as $$
+CREATE OR REPLACE FUNCTION CDB_KMeans(
+  query text,
+  no_clusters integer,
+  no_init integer default 20
+)
+RETURNS TABLE(
+  cartodb_id integer,
+  cluster_no integer
+) AS $$
 
-    from crankshaft.clustering import Kmeans
-    kmeans = Kmeans()
-    return kmeans.spatial(query, no_clusters, no_init)
+from crankshaft.clustering import Kmeans
+kmeans = Kmeans()
+return kmeans.spatial(query, no_clusters, no_init)
 
 $$ LANGUAGE plpythonu;
 
 -- Non-spatial k-means clustering
 -- query: sql query to retrieve all the needed data
 -- colnames: text array of column names for doing the clustering analysis
+-- no_clusters: number of requested clusters
 -- standardize: whether to scale variables to a mean of zero and a standard
 --              deviation of 1
 -- id_colname: name of the id column
@@ -19,24 +27,32 @@ $$ LANGUAGE plpythonu;
 CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
   query TEXT,
   colnames TEXT[],
-  num_clusters INTEGER,
+  no_clusters INTEGER,
   standardize BOOLEAN DEFAULT true,
   id_col TEXT DEFAULT 'cartodb_id'
 )
-RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
+RETURNS TABLE(
+  cluster_label text,
+  cluster_center json,
+  silhouettes numeric,
+  inertia numeric,
+  rowid bigint
+) AS $$
 
-    from crankshaft.clustering import Kmeans
-    kmeans = Kmeans()
-    return kmeans.nonspatial(query, colnames, num_clusters,
-                             standardize=standardize,
-                             id_col=id_col)
+from crankshaft.clustering import Kmeans
+kmeans = Kmeans()
+return kmeans.nonspatial(query, colnames, no_clusters,
+                         standardize=standardize,
+                         id_col=id_col)
 $$ LANGUAGE plpythonu;
 
 
-
-CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)
-RETURNS Numeric[] AS
-$$
+CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(
+  state Numeric[],
+  the_geom GEOMETRY(Point, 4326),
+  weight NUMERIC
+)
+RETURNS Numeric[] AS $$
 DECLARE
     newX NUMERIC;
     newY NUMERIC;
@@ -56,6 +72,7 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 
+
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[])
 RETURNS GEOMETRY AS
 $$

From 001062f66020340de1adfce326e8b592b29552b4 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 13:02:55 -0500
Subject: [PATCH 15/24] adds inertia as an output column

---
 doc/11_kmeans.md                              | 45 ++++++++++++++-----
 .../crankshaft/clustering/kmeans.py           | 24 +++++-----
 2 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md
index cd4a2fa..4a32be7 100644
--- a/doc/11_kmeans.md
+++ b/doc/11_kmeans.md
@@ -1,5 +1,7 @@
 ## K-Means Functions
 
+k-means clustering is a popular technique for finding clusters in data by minimizing the intra-cluster 'distance' and maximizing the inter-cluster 'distance'. The distance is defined in the parameter space of the variables entered.
+
 ### CDB_KMeans(subquery text, no_clusters INTEGER)
 
 This function attempts to find `no_clusters` clusters within the input data based on the geographic distribution. It will return a table with ids and the cluster classification of each point input assuming `the_geom` is not null-valued. If `the_geom` is null-valued, the point will not be considered in the analysis.
@@ -9,7 +11,7 @@ This function attempts to find `no_clusters` clusters within the input data base
 | Name | Type | Description |
 |------|------|-------------|
 | subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments |
-| no\_clusters | INTEGER | The number of clusters to try and find |
+| no\_clusters | INTEGER | The number of clusters to find |
 
 #### Returns
 
@@ -27,10 +29,11 @@ A table with the following columns.
 SELECT
     customers.*,
     km.cluster_no
-    FROM
-      cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) As km,
-      customers
-    WHERE customers.cartodb_id = km.cartodb_id
+FROM
+    cdb_crankshaft.CDB_KMeans('SELECT * from customers' , 6) As km,
+    customers
+WHERE
+    customers.cartodb_id = km.cartodb_id
 ```
 
 ### CDB_WeightedMean(subquery text, weight_column text, category_column text)
@@ -58,13 +61,13 @@ A table with the following columns.
 
 ```sql
 SELECT
-  ST_Transform(the_geom, 3857) As the_geom_webmercator,
-  class
+    ST_Transform(km.the_geom, 3857) As the_geom_webmercator,
+    km.class
 FROM
-  cdb_crankshaft.CDB_WeightedMean(
-    'SELECT *, customer_value FROM customers',
-    'customer_value',
-    'cluster_no')
+    cdb_crankshaft.CDB_WeightedMean(
+        'SELECT *, customer_value FROM customers',
+        'customer_value',
+        'cluster_no') As km
 ```
 
 ## CDB_KMeansNonspatial(subquery text, colnames text[], no_clusters int)
@@ -80,7 +83,7 @@ As a standard machine learning method, k-means clustering is an unsupervised lea
 | query | TEXT | SQL query to expose the data to be used in the analysis (e.g., `SELECT * FROM iris_data`). It should contain at least the columns specified in `colnames` and the `id_colname`. |
 | colnames | TEXT[] | Array of columns to be used in the analysis (e.g., `Array['petal_width', 'sepal_length', 'petal_length']`). |
 | no\_clusters | INTEGER | Number of clusters for the classification of the data |
-| id_col (optional) | TEXT | The id column (default: 'cartodb_id') for identifying rows |
+| id\_col (optional) | TEXT | The id column (default: 'cartodb_id') for identifying rows |
 | standarize (optional) | BOOLEAN | Setting this to true (default) standardizes the data to have a mean at zero and a standard deviation of 1 |
 
 ### Returns
@@ -92,8 +95,26 @@ A table with the following columns.
 | cluster_label | TEXT | Label that a cluster belongs to, number from 0 to `no_clusters - 1`. |
 | cluster_center | JSON | Center of the cluster that a row belongs to. The keys of the JSON object are the `colnames`, with values that are the center of the respective cluster |
 | silhouettes | NUMERIC | [Silhouette score](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.silhouette_score.html#sklearn.metrics.silhouette_score) of the cluster label |
+| inertia | NUMERIC | Sum of squared distances of samples to their closest cluster center |
 | rowid | BIGINT | id of the original row for associating back with the original data |
 
+### Example Usage
+
+```sql
+SELECT
+    customers.*,
+    km.cluster_label,
+    km.cluster_center,
+    km.silhouettes
+FROM
+    cdb_crankshaft.CDB_KMeansNonspatial(
+        'SELECT * FROM customers',
+        Array['customer_value', 'avg_amt_spent', 'home_median_income'],
+        7) As km,
+    customers
+WHERE
+    customers.cartodb_id = km.rowid
+```
 
 ### Resources
 
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index fe6831f..559d304 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -34,7 +34,7 @@ class Kmeans:
     def nonspatial(self, subquery, colnames, no_clusters=5,
                    standardize=True, id_col='cartodb_id'):
         """
-            Inputs:
+        Arguments:
             query (string): A SQL query to retrieve the data required to do the
                             k-means clustering analysis, like so:
                             SELECT * FROM iris_flower_data
@@ -46,7 +46,7 @@ class Kmeans:
             no_clusters (int): number of clusters (greater than zero)
             id_col (string): name of the input id_column
 
-            Output:
+        Returns:
             A list of tuples with the following columns:
             cluster labels: a label for the cluster that the row belongs to
             centers: center of the cluster that this row belongs to
@@ -57,19 +57,20 @@ class Kmeans:
         import json
         from sklearn import metrics
 
-        # TODO: need a random seed?
-        params = {"colnames": colnames,
-                  "subquery": subquery,
-                  "id_col": id_col}
+        params = {
+            "colnames": colnames,
+            "subquery": subquery,
+            "id_col": id_col
+        }
 
         data = self.data_provider.get_nonspatial_kmeans(params)
 
         # fill array with values for k-means clustering
         if standardize:
             cluster_columns = _scale_data(
-              _extract_columns(data, len(colnames)))
+              _extract_columns(data))
         else:
-            cluster_columns = _extract_columns(data, len(colnames))
+            cluster_columns = _extract_columns(data)
 
         kmeans = KMeans(n_clusters=no_clusters,
                         random_state=0).fit(cluster_columns)
@@ -84,18 +85,19 @@ class Kmeans:
         return zip(kmeans.labels_,
                    centers,
                    silhouettes,
+                   [kmeans.inertia_] * kmeans.labels_.shape[0],
                    data[0]['rowid'])
 
 
 # -- Preprocessing steps
 
-def _extract_columns(data, n_cols):
+def _extract_columns(data):
     """
         Extract the features from the query and pack them into a NumPy array
         data (list of dicts): result of the kmeans request
-        id_col_name (string): name of column which has the row id (not a
-                              feature of the analysis)
     """
+    # number of columns minus rowid column
+    n_cols = len(data) - 1
     return np.array([data[0]['arr_col{0}'.format(i+1)]
                      for i in xrange(n_cols)],
                     dtype=float).T

From 49a317ae8ec0a63c734effa2797ce9be78824817 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 13:29:36 -0500
Subject: [PATCH 16/24] syntax updates / consistency

---
 src/pg/sql/11_kmeans.sql | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 1c34b75..4fe9c65 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -1,13 +1,13 @@
 -- Spatial k-means clustering
 
 CREATE OR REPLACE FUNCTION CDB_KMeans(
-  query text,
-  no_clusters integer,
-  no_init integer default 20
+  query TEXT,
+  no_clusters INTEGER,
+  no_init INTEGER DEFAULT 20
 )
 RETURNS TABLE(
-  cartodb_id integer,
-  cluster_no integer
+  cartodb_id INTEGER,
+  cluster_no INTEGER
 ) AS $$
 
 from crankshaft.clustering import Kmeans
@@ -48,7 +48,7 @@ $$ LANGUAGE plpythonu;
 
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(
-  state Numeric[],
+  state NUMERIC[],
   the_geom GEOMETRY(Point, 4326),
   weight NUMERIC
 )
@@ -73,7 +73,7 @@ END
 $$ LANGUAGE plpgsql;
 
 
-CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[])
+CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state NUMERIC[])
 RETURNS GEOMETRY AS
 $$
 BEGIN

From 20104c2df953d9bb2cac1ee91e473aaa9156d22e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 13:35:00 -0500
Subject: [PATCH 17/24] adds sql tests for nonspatial kmeans

---
 src/pg/test/expected/11_kmeans_test.out | 31 +++++++++++++++++++++++--
 src/pg/test/sql/11_kmeans_test.sql      | 29 +++++++++++++++++++++--
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
index 8c6ffa1..78ccfa0 100644
--- a/src/pg/test/expected/11_kmeans_test.out
+++ b/src/pg/test/expected/11_kmeans_test.out
@@ -1,10 +1,37 @@
 \pset format unaligned
 \set ECHO all
-SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
+SELECT
+    count(DISTINCT cluster_no) as clusters
+FROM
+    cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
 clusters
 2
 (1 row)
-SELECT count(*) clusters from (select  cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p;
+SELECT
+    count(*) clusters
+FROM (
+    SELECT
+        cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC),
+        code
+    FROM ppoints
+    GROUP BY code
+) p;
 clusters
 52
 (1 row)
+SELECT
+    cluster_label,
+    cluster_center,
+    silhouettes,
+    inertia,
+    rowid
+FROM cdb_crankshaft.CDB_KMeansNonspatial(
+    'select unnest(Array[1, 1, 10, 10]) as col1, unnest(100, 100, 2, 2) as col2 from ppoints',
+    Array['col1', 'col2']::text[],
+    2);
+cluster_label|cluster_center|silhouettes|inertia|rowid
+0|'{"col1":1,"col2":100}'||0|1
+0|'{"col1":1,"col2":100}'||0|2
+1|'{"col1":10,"col2":2}'||0|3
+1|'{"col1":10,"col2":2}'||0|4
+(4 rows)
diff --git a/src/pg/test/sql/11_kmeans_test.sql b/src/pg/test/sql/11_kmeans_test.sql
index 2298b85..0b598c4 100644
--- a/src/pg/test/sql/11_kmeans_test.sql
+++ b/src/pg/test/sql/11_kmeans_test.sql
@@ -1,6 +1,31 @@
 \pset format unaligned
 \set ECHO all
 
-SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
+-- spatial kmeans
+SELECT
+    count(DISTINCT cluster_no) as clusters
+FROM
+    cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
 
-SELECT count(*) clusters from (select  cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p;
+-- weighted mean
+SELECT
+    count(*) clusters
+FROM (
+    SELECT
+        cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC),
+        code
+    FROM ppoints
+    GROUP BY code
+) p;
+
+-- nonspatial kmeans
+SELECT
+    cluster_label,
+    cluster_center,
+    silhouettes,
+    inertia,
+    rowid
+FROM cdb_crankshaft.CDB_KMeansNonspatial(
+    'select unnest(Array[1, 1, 10, 10]) as col1, unnest(100, 100, 2, 2) as col2 from ppoints',
+    Array['col1', 'col2']::text[],
+    2);

From b0e3f38f1e02a86146a44819df894b1148b1c365 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 14:28:20 -0500
Subject: [PATCH 18/24] correctly finds the number of columns

---
 src/py/crankshaft/crankshaft/clustering/kmeans.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index fe259ee..6d22d44 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -97,7 +97,7 @@ def _extract_columns(data):
         data (list of dicts): result of the kmeans request
     """
     # number of columns minus rowid column
-    n_cols = len(data) - 1
+    n_cols = len(data[0]) - 1
     return np.array([data[0]['arr_col{0}'.format(i+1)]
                      for i in xrange(n_cols)],
                     dtype=float).T

From 18fbc2fa9eb9903a39938c0ed6fe79b8446d73b8 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 14:39:24 -0500
Subject: [PATCH 19/24] updates sql query / fixes error

---
 src/pg/test/expected/11_kmeans_test.out | 2 +-
 src/pg/test/sql/11_kmeans_test.sql      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
index 78ccfa0..4840c57 100644
--- a/src/pg/test/expected/11_kmeans_test.out
+++ b/src/pg/test/expected/11_kmeans_test.out
@@ -26,7 +26,7 @@ SELECT
     inertia,
     rowid
 FROM cdb_crankshaft.CDB_KMeansNonspatial(
-    'select unnest(Array[1, 1, 10, 10]) as col1, unnest(100, 100, 2, 2) as col2 from ppoints',
+    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2 FROM ppoints',
     Array['col1', 'col2']::text[],
     2);
 cluster_label|cluster_center|silhouettes|inertia|rowid
diff --git a/src/pg/test/sql/11_kmeans_test.sql b/src/pg/test/sql/11_kmeans_test.sql
index 0b598c4..7ce6310 100644
--- a/src/pg/test/sql/11_kmeans_test.sql
+++ b/src/pg/test/sql/11_kmeans_test.sql
@@ -26,6 +26,6 @@ SELECT
     inertia,
     rowid
 FROM cdb_crankshaft.CDB_KMeansNonspatial(
-    'select unnest(Array[1, 1, 10, 10]) as col1, unnest(100, 100, 2, 2) as col2 from ppoints',
+    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2 FROM ppoints',
     Array['col1', 'col2']::text[],
     2);

From 04f290cbad3ddc241bf035d221d463f64b269853 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 15:06:44 -0500
Subject: [PATCH 20/24] finalizes test query

---
 src/pg/test/expected/11_kmeans_test.out | 12 ++++++------
 src/pg/test/sql/11_kmeans_test.sql      |  9 ++++++---
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
index 4840c57..5b22f1b 100644
--- a/src/pg/test/expected/11_kmeans_test.out
+++ b/src/pg/test/expected/11_kmeans_test.out
@@ -26,12 +26,12 @@ SELECT
     inertia,
     rowid
 FROM cdb_crankshaft.CDB_KMeansNonspatial(
-    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2 FROM ppoints',
+    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2, unnest(Array[1, 2, 3, 4]) As cartodb_id',
     Array['col1', 'col2']::text[],
     2);
-cluster_label|cluster_center|silhouettes|inertia|rowid
-0|'{"col1":1,"col2":100}'||0|1
-0|'{"col1":1,"col2":100}'||0|2
-1|'{"col1":10,"col2":2}'||0|3
-1|'{"col1":10,"col2":2}'||0|4
+cluster_label|cc_col1|cc_col2|silhouettes|inertia|rowid
+t|-1.0|1.0|1.0|0.0|1
+t|-1.0|1.0|1.0|0.0|2
+t|1.0|-1.0|1.0|0.0|3
+t|1.0|-1.0|1.0|0.0|4
 (4 rows)
diff --git a/src/pg/test/sql/11_kmeans_test.sql b/src/pg/test/sql/11_kmeans_test.sql
index 7ce6310..a44e315 100644
--- a/src/pg/test/sql/11_kmeans_test.sql
+++ b/src/pg/test/sql/11_kmeans_test.sql
@@ -20,12 +20,15 @@ FROM (
 
 -- nonspatial kmeans
 SELECT
-    cluster_label,
-    cluster_center,
+    cluster_label::int in (0, 1) As cluster_label,
+    cluster_center::json->>'col1' As cc_col1,
+    cluster_center::json->>'col2' As cc_col2,
     silhouettes,
     inertia,
     rowid
 FROM cdb_crankshaft.CDB_KMeansNonspatial(
-    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2 FROM ppoints',
+    'SELECT unnest(Array[1, 1, 10, 10]) As col1, ' ||
+    'unnest(Array[100, 100, 2, 2]) As col2, ' ||
+    'unnest(Array[1, 2, 3, 4]) As cartodb_id ',
     Array['col1', 'col2']::text[],
     2);

From cfd988c33897bf77c17bbf0f8526a6b7251c900c Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 15:14:38 -0500
Subject: [PATCH 21/24] uses exact same query text for expectation

---
 src/pg/test/expected/11_kmeans_test.out | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
index 5b22f1b..1a0077c 100644
--- a/src/pg/test/expected/11_kmeans_test.out
+++ b/src/pg/test/expected/11_kmeans_test.out
@@ -20,13 +20,16 @@ clusters
 52
 (1 row)
 SELECT
-    cluster_label,
-    cluster_center,
+    cluster_label::int in (0, 1) As cluster_label,
+    cluster_center::json->>'col1' As cc_col1,
+    cluster_center::json->>'col2' As cc_col2,
     silhouettes,
     inertia,
     rowid
 FROM cdb_crankshaft.CDB_KMeansNonspatial(
-    'SELECT unnest(Array[1, 1, 10, 10]) As col1, unnest(Array[100, 100, 2, 2]) As col2, unnest(Array[1, 2, 3, 4]) As cartodb_id',
+    'SELECT unnest(Array[1, 1, 10, 10]) As col1, ' ||
+    'unnest(Array[100, 100, 2, 2]) As col2, ' ||
+    'unnest(Array[1, 2, 3, 4]) As cartodb_id ',
     Array['col1', 'col2']::text[],
     2);
 cluster_label|cc_col1|cc_col2|silhouettes|inertia|rowid

From 604f20bb21efb341917914a9ca487ee0b9ad52c5 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 15:22:14 -0500
Subject: [PATCH 22/24] grrr adds comments to test expectation

---
 src/pg/test/expected/11_kmeans_test.out | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out
index 1a0077c..85b8b13 100644
--- a/src/pg/test/expected/11_kmeans_test.out
+++ b/src/pg/test/expected/11_kmeans_test.out
@@ -1,5 +1,6 @@
 \pset format unaligned
 \set ECHO all
+-- spatial kmeans
 SELECT
     count(DISTINCT cluster_no) as clusters
 FROM
@@ -7,6 +8,7 @@ FROM
 clusters
 2
 (1 row)
+-- weighted mean
 SELECT
     count(*) clusters
 FROM (
@@ -19,6 +21,7 @@ FROM (
 clusters
 52
 (1 row)
+-- nonspatial kmeans
 SELECT
     cluster_label::int in (0, 1) As cluster_label,
     cluster_center::json->>'col1' As cc_col1,

From 068f43de107ffbe1cf1188558b59c10365266cd9 Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Tue, 9 Jan 2018 15:45:03 -0500
Subject: [PATCH 23/24] adds test for exception if no data is present

---
 src/py/crankshaft/test/test_clustering_kmeans.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py
index 3756b7e..c118d34 100644
--- a/src/py/crankshaft/test/test_clustering_kmeans.py
+++ b/src/py/crankshaft/test/test_clustering_kmeans.py
@@ -71,7 +71,6 @@ class KMeansNonspatialTest(unittest.TestCase):
         random_seeds.set_random_seeds(1234)
         kmeans = Kmeans(FakeDataProvider(data_raw))
         clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2)
-        print str([c[0] for c in clusters])
 
         cl1 = clusters[0][0]
         cl2 = clusters[3][0]
@@ -81,3 +80,8 @@ class KMeansNonspatialTest(unittest.TestCase):
                 self.assertEqual(val[0], cl1)
             else:
                 self.assertEqual(val[0], cl2)
+
+        # raises exception for no data
+        with self.assertRaises(Exception):
+            kmeans = Kmeans(FakeDataProvider([]))
+            kmeans.nonspatial('subquery', ['col1', 'col2'], 2)

From e5285a27008fd279afd2cf5767d930e183614cec Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Wed, 10 Jan 2018 10:57:51 -0500
Subject: [PATCH 24/24] adds parallel marker for plpgsql function

---
 src/pg/sql/11_kmeans.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql
index 8ca7840..b0362b8 100644
--- a/src/pg/sql/11_kmeans.sql
+++ b/src/pg/sql/11_kmeans.sql
@@ -44,7 +44,7 @@ kmeans = Kmeans()
 return kmeans.nonspatial(query, colnames, no_clusters,
                          standardize=standardize,
                          id_col=id_col)
-$$ LANGUAGE plpythonu;
+$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
 
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(