From 798e754dfb091789e84ca03476acd0813785dd5e Mon Sep 17 00:00:00 2001
From: Andy Eschbacher <andy.eschbacher@gmail.com>
Date: Mon, 5 Dec 2016 17:14:36 -0500
Subject: [PATCH] stubs in kmeans non-spatial

---
 .../crankshaft/analysis_data_provider.py      |  9 ++-
 .../crankshaft/clustering/kmeans.py           | 72 +++++++++++++++++++
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py
index cbc27bc..34512ae 100644
--- a/src/py/crankshaft/crankshaft/analysis_data_provider.py
+++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py
@@ -44,8 +44,15 @@ class AnalysisDataProvider:
             plpy.error('Analysis failed: %s' % e)
             return pu.empty_zipped_array(2)
 
-    def get_nonspatial_kmeans(self, query):
+    def get_nonspatial_kmeans(self, params):
         """fetch data for non-spatial kmeans"""
+        query = '''
+            SELECT {cols}, array_agg({id_col}) As rowid
+            FROM ({subquery}) As a
+        '''.format(subquery=subquery,
+                   id_col=id_col,
+                   cols=', '.join(['array_agg({0}) As arr_{0}'.format(c)
+                                   for c in params[colnames]]))
         try:
             data = plpy.execute(query)
             return data
diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py
index 1e49115..84d3e0a 100644
--- a/src/py/crankshaft/crankshaft/clustering/kmeans.py
+++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py
@@ -30,3 +30,75 @@ class Kmeans:
         km = KMeans(n_clusters=no_clusters, n_init=no_init)
         labels = km.fit_predict(zip(xs, ys))
         return zip(ids, labels)
+
+    def nonspatial(self, subquery, colnames, num_clusters=5,
+                   id_col='cartodb_id', standarize=True):
+        """
+            query (string): A SQL query to retrieve the data required to do the
+                            k-means clustering analysis, like so:
+                            SELECT * FROM iris_flower_data
+            colnames (list): a list of the column names which contain the data
+                             of interest, like so: ["sepal_width",
+                                                    "petal_width",
+                                                    "sepal_length",
+                                                    "petal_length"]
+            num_clusters (int): number of clusters (greater than zero)
+            id_col (string): name of the input id_column
+        """
+        import json
+        from sklearn import metrics
+
+        out_id_colname = 'rowids'
+        # TODO: need a random seed?
+        params = {"cols": colnames,
+                  "subquery": subquery,
+                  "id_col": id_col}
+
+        data = self.query_runner.get_nonspatial_kmeans(params, standarize)
+
+        # fill array with values for k-means clustering
+        if standarize:
+            cluster_columns = _scale_data(
+              _extract_columns(data, colnames))
+        else:
+            cluster_columns = _extract_columns(data, colnames)
+
+        print str(cluster_columns)
+        # TODO: decide on optimal parameters for most cases
+        #       Are there ways of deciding parameters based on inputs?
+        kmeans = KMeans(n_clusters=num_clusters,
+                        random_state=0).fit(cluster_columns)
+
+        centers = [json.dumps(dict(zip(colnames, c)))
+                   for c in kmeans.cluster_centers_[kmeans.labels_]]
+
+        silhouettes = metrics.silhouette_samples(cluster_columns,
+                                                 kmeans.labels_,
+                                                 metric='sqeuclidean')
+
+        return zip(kmeans.labels_,
+                   centers,
+                   silhouettes,
+                   data[0][out_id_colname])
+
+
+# -- Preprocessing steps
+
+def _extract_columns(data, colnames):
+    """
+        Extract the features from the query and pack them into a NumPy array
+        data (list of dicts): result of the kmeans request
+        id_col_name (string): name of column which has the row id (not a
+                              feature of the analysis)
+    """
+    return np.array([data[0]['arr_{}'.format(c)] for c in colnames],
+                    dtype=float).T
+
+
+def _scale_data(features):
+    """
+        Scale all input columns to center on 0 with a standard devation of 1
+        features (numpy matrix): features of dimension (n_features, n_samples)
+    """
+    from sklearn.preprocessing import StandardScaler
+    return StandardScaler().fit_transform(features)