remove unneeded commented code

src/pg/sql/11_kmeans.sql

@@ -1,10 +1,29 @@
-CREATE OR REPLACE FUNCTION  CDB_KMeans(query text, no_clusters integer,no_init integer default 20)
+-- Spatial k-means clustering
+
+CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer, no_init integer default 20)
 RETURNS table (cartodb_id integer, cluster_no integer) as $$
 
     from crankshaft.clustering import kmeans
-    return kmeans(query,no_clusters,no_init)
+    return kmeans(query, no_clusters, no_init)
 
-$$ language plpythonu;
+$$ LANGUAGE plpythonu;
+
+-- Non-spatial k-means clustering
+-- query: sql query to retrieve all the needed data
+
+CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial(
+  query TEXT,
+  colnames TEXT[],
+  num_clusters INTEGER,
+  id_colname TEXT DEFAULT 'cartodb_id',
+  standarize BOOLEAN DEFAULT true
+)
+RETURNS TABLE(cluster_label text, cluster_center json, silhouettes numeric, rowid bigint) AS $$
+
+    from crankshaft.clustering import kmeans_nonspatial
+    return kmeans_nonspatial(query, colnames, num_clusters,
+                             id_colname, standarize)
+$$ LANGUAGE plpythonu;
 
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)

src/py/crankshaft/crankshaft/clustering/kmeans.py

@@ -1,18 +1,109 @@
 from sklearn.cluster import KMeans
 import plpy
+import numpy as np
+
 
 def kmeans(query, no_clusters, no_init=20):
-    data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids,
+    """
-        array_agg(ST_X(the_geom) order by cartodb_id) xs,
+        find centers based on clusteres of latitude/longitude pairs
-        array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a
+        query: SQL query that has a WGS84 geometry (the_geom)
-        where the_geom is not null
+    """
-    '''.format(query=query))
+    full_query = ("SELECT array_agg(cartodb_id ORDER BY cartodb_id) as ids,"
+                  "array_agg(ST_X(the_geom) ORDER BY cartodb_id) xs,"
+                  "array_agg(ST_Y(the_geom) ORDER BY cartodb_id) ys "
+                  "FROM ({query}) As a "
+                  "WHERE the_geom IS NOT NULL").format(query=query)
+    try:
+        data = plpy.execute(full_query)
+    except plpy.SPIError, err:
+        plpy.error("k-means (spatial) cluster analysis failed: %s" % err)
 
-    xs  = data[0]['xs']
+    # Unpack query response
-    ys  = data[0]['ys']
+    xs = data[0]['xs']
+    ys = data[0]['ys']
     ids = data[0]['ids']
 
-    km = KMeans(n_clusters= no_clusters, n_init=no_init)
+    km = KMeans(n_clusters=no_clusters, n_init=no_init)
-    labels = km.fit_predict(zip(xs,ys))
+    labels = km.fit_predict(zip(xs, ys))
-    return zip(ids,labels)
+    return zip(ids, labels)
 
+
+def kmeans_nonspatial(query, colnames, num_clusters=5,
+                      id_col='cartodb_id', standarize=True):
+    """
+        query (string): A SQL query to retrieve the data required to do the
+                        k-means clustering analysis, like so:
+                        SELECT * FROM iris_flower_data
+        colnames (list): a list of the column names which contain the data of
+                         interest, like so: ["sepal_width", "petal_width",
+                                             "sepal_length", "petal_length"]
+        num_clusters (int): number of clusters (greater than zero)
+        id_col (string): name of the input id_column
+    """
+    import json
+    from sklearn import metrics
+
+    out_id_colname = 'rowids'
+    # TODO: need a random seed?
+
+    full_query = '''
+        SELECT {cols}, array_agg({id_col}) As {out_id_colname}
+        FROM ({query}) As a
+    '''.format(query=query,
+               id_col=id_col,
+               out_id_colname=out_id_colname,
+               cols=', '.join(['array_agg({0}) As col{1}'.format(val, idx)
+                               for idx, val in enumerate(colnames)]))
+
+    try:
+        db_resp = plpy.execute(full_query)
+    except plpy.SPIError, err:
+        plpy.error("k-means (non-spatial) cluster analysis failed: %s" % err)
+
+    # fill array with values for k-means clustering
+    if standarize:
+        cluster_columns = _scale_data(
+          _extract_columns(db_resp, out_id_colname))
+    else:
+        cluster_columns = _extract_columns(db_resp, out_id_colname)
+
+    # TODO: decide on optimal parameters for most cases
+    #       Are there ways of deciding parameters based on inputs?
+    kmeans = KMeans(n_clusters=num_clusters,
+                    random_state=0).fit(cluster_columns)
+
+    centers = [json.dumps(dict(zip(colnames, c)))
+               for c in kmeans.cluster_centers_[kmeans.labels_]]
+
+    silhouettes = metrics.silhouette_samples(cluster_columns,
+                                             kmeans.labels_,
+                                             metric='sqeuclidean')
+
+    return zip(kmeans.labels_,
+               centers,
+               silhouettes,
+               db_resp[0][out_id_colname])
+
+
+def _extract_columns(db_resp, id_col_name):
+    """
+        Extract the features from the query and pack them into a NumPy array
+        db_resp (plpy data object): result of the kmeans request
+        id_col_name (string): name of column which has the row id (not a
+                              feature of the analysis)
+    """
+    return np.array([db_resp[0][c] for c in db_resp.colnames()
+                     if c != id_col_name],
+                    dtype=float).T
+
+# -- Preprocessing steps
+
+
+def _scale_data(features):
+    """
+        Scale all input columns to center on 0 with a standard devation of 1
+
+        features (numpy matrix): features of dimension (n_features, n_samples)
+    """
+    from sklearn.preprocessing import StandardScaler
+    return StandardScaler().fit_transform(features)

src/py/crankshaft/test/helper.py

@@ -2,6 +2,7 @@ import unittest
 
 from mock_plpy import MockPlPy
 plpy = MockPlPy()
+from mock_plpy import MockDBResponse
 
 import sys
 sys.modules['plpy'] = plpy

src/py/crankshaft/test/mock_plpy.py

@@ -1,12 +1,13 @@
 import re
 
+
 class MockCursor:
     def __init__(self, data):
         self.cursor_pos = 0
         self.data = data
 
     def fetch(self, batch_size):
-        batch = self.data[self.cursor_pos : self.cursor_pos + batch_size]
+        batch = self.data[self.cursor_pos:self.cursor_pos + batch_size]
         self.cursor_pos += batch_size
         return batch
 
@@ -45,8 +46,22 @@ class MockPlPy:
         data = self.execute(query)
         return MockCursor(data)
 
-    def execute(self, query): # TODO: additional arguments
+    # TODO: additional arguments
-       for result in self.results:
+    def execute(self, query):
-          if result[0].match(query):
+        for result in self.results:
-            return result[1]
+            if result[0].match(query):
-       return []
+                return result[1]
+        return []
+
+
+class MockDBResponse:
+    def __init__(self, data, colnames=None):
+        self.data = data
+        if colnames is None:
+            self.colnames = data[0].keys()
+        else:
+            self.colnames = colnames
+
+
+    def colnames(self):
+        return self.colnames

src/py/crankshaft/test/test_cluster_kmeans.py

@@ -1,38 +0,0 @@
-import unittest
-import numpy as np
-
-
-# from mock_plpy import MockPlPy
-# plpy = MockPlPy()
-#
-# import sys
-# sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
-import numpy as np
-import crankshaft.clustering as cc
-import crankshaft.pysal_utils as pu
-from crankshaft import random_seeds
-import json
-
-class KMeansTest(unittest.TestCase):
-    """Testing class for Moran's I functions"""
-
-    def setUp(self):
-        plpy._reset()
-        self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read())
-        self.params = {"subquery": "select * from table",
-                       "no_clusters": "10"
-                       }
-
-    def test_kmeans(self):
-        data = self.cluster_data
-        plpy._define_result('select' ,data)
-        clusters = cc.kmeans('subquery', 2)
-        labels  = [a[1] for a in clusters]
-        c1 = [a for a in clusters if a[1]==0]
-        c2 = [a for a in clusters if a[1]==1]
-
-        self.assertEqual(len(np.unique(labels)),2)
-        self.assertEqual(len(c1),20)
-        self.assertEqual(len(c2),20)
-

src/py/crankshaft/test/test_clustering_kmeans.py

@@ -0,0 +1,73 @@
+import unittest
+import numpy as np
+
+
+# from mock_plpy import MockPlPy
+# plpy = MockPlPy()
+#
+# import sys
+# sys.modules['plpy'] = plpy
+from helper import plpy, fixture_file, MockDBResponse
+import crankshaft.clustering as cc
+import json
+from collections import OrderedDict
+
+
+class KMeansTest(unittest.TestCase):
+    """Testing class for k-means spatial"""
+
+    def setUp(self):
+        plpy._reset()
+        self.cluster_data = json.loads(
+          open(fixture_file('kmeans.json')).read())
+        self.params = {"subquery": "select * from table",
+                       "no_clusters": "10"}
+
+    def test_kmeans(self):
+        """
+        """
+        data = [{'xs': d['xs'],
+                 'ys': d['ys'],
+                 'ids': d['ids']} for d in self.cluster_data]
+
+        plpy._define_result('select', data)
+        clusters = cc.kmeans('subquery', 2)
+        labels = [a[1] for a in clusters]
+        c1 = [a for a in clusters if a[1] == 0]
+        c2 = [a for a in clusters if a[1] == 1]
+
+        self.assertEqual(len(np.unique(labels)), 2)
+        self.assertEqual(len(c1), 20)
+        self.assertEqual(len(c2), 20)
+
+
+class KMeansNonspatialTest(unittest.TestCase):
+    """Testing class for k-means non-spatial"""
+
+    def setUp(self):
+        plpy._reset()
+
+        self.params = {"subquery": "SELECT * FROM TABLE",
+                       "n_clusters": 5}
+
+    def test_kmeans_nonspatial(self):
+        """
+            test for k-means non-spatial
+        """
+        data_raw = [OrderedDict([("col1", [1, 1, 1, 4, 4, 4]),
+                                 ("col2", [2, 4, 0, 2, 4, 0]),
+                                 ("rowids", [1, 2, 3, 4, 5, 6])])]
+
+        data_obj = MockDBResponse(data_raw, [k for k in data_raw[0]
+                                             if k != 'rowids'])
+        plpy._define_result('select', data_obj)
+        clusters = cc.kmeans_nonspatial('subquery', ['col1', 'col2'], 4)
+
+        cl1 = clusters[0][1]
+        cl2 = clusters[3][1]
+
+        for idx, val in enumerate(clusters):
+            if idx < 3:
+                self.assertEqual(val[1], cl1)
+            else:
+                self.assertEqual(val[1], cl2)

src/py/crankshaft/test/test_clustering_moran.py

@@ -7,13 +7,13 @@ import numpy as np
 #
 # import sys
 # sys.modules['plpy'] = plpy
-from helper import plpy, fixture_file
+from helper import plpy, fixture_file, MockDBResponse
 
 import crankshaft.clustering as cc
 import crankshaft.pysal_utils as pu
 from crankshaft import random_seeds
 import json
-
+from collections import OrderedDict
 
 class MoranTest(unittest.TestCase):
     """Testing class for Moran's I functions"""
@@ -58,11 +58,14 @@ class MoranTest(unittest.TestCase):
 
     def test_moran_local(self):
         """Test Moran's I local"""
-        data = [{'id': d['id'],
+        data = [OrderedDict([('id', d['id']),
-                 'attr1': d['value'],
+                             ('attr1', d['value']),
-                 'neighbors': d['neighbors']} for d in self.neighbors_data]
+                             ('neighbors', d['neighbors'])])
+                for d in self.neighbors_data]
 
-        plpy._define_result('select', data)
+        db_resp = MockDBResponse(data)
+
+        plpy._define_result('select', db_resp)
         random_seeds.set_random_seeds(1234)
         result = cc.moran_local('subquery', 'value',
                                 'knn', 5, 99, 'the_geom', 'cartodb_id')