diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md index 6fe571a..97d85aa 100644 --- a/doc/11_kmeans.md +++ b/doc/11_kmeans.md @@ -1,17 +1,17 @@ ## K-Means Functions +k-means clustering is a popular technique for finding clusters in data by minimizing the intra-cluster 'distance' and maximizing the inter-cluster 'distance'. The distance is defined in the parameter space of the variables entered. + ### CDB_KMeans(subquery text, no_clusters integer) -This function attempts to find n clusters within the input data. It will return a table to CartoDB ids and -the number of the cluster each point in the input was assigend to. - +This function attempts to find `no_clusters` clusters within the input data based on the geographic distribution. It will return a table with ids and the cluster classification of each point input assuming `the_geom` is not null-valued. If `the_geom` is null-valued, the point will not be considered in the analysis. #### Arguments | Name | Type | Description | |------|------|-------------| | subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | -| no\_clusters | INTEGER | The number of clusters to try and find | +| no\_clusters | INTEGER | The number of clusters to find | #### Returns @@ -19,8 +19,8 @@ A table with the following columns. | Column Name | Type | Description | |-------------|------|-------------| -| cartodb\_id | INTEGER | The CartoDB id of the row in the input table.| -| cluster\_no | INTEGER | The cluster that this point belongs to. | +| cartodb\_id | INTEGER | The row id of the row from the input table | +| cluster\_no | INTEGER | The cluster that this point belongs to | #### Example Usage @@ -30,7 +30,8 @@ SELECT customers.*, km.cluster_no FROM - cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) km, customers_3 + cdb_crankshaft.CDB_KMeans('SELECT * from customers' , 6) As km, + customers WHERE customers.cartodb_id = km.cartodb_id ``` @@ -60,11 +61,62 @@ A table with the following columns. ```sql SELECT - ST_Transform(m.the_geom, 3857) AS the_geom_webmercator, - m.class + ST_Transform(km.the_geom, 3857) As the_geom_webmercator, + km.class FROM - cdb_crankshaft.cdb_WeightedMean( - 'SELECT * FROM customers', + cdb_crankshaft.CDB_WeightedMean( + 'SELECT *, customer_value FROM customers', 'customer_value', - 'cluster_no') AS m + 'cluster_no') As km ``` + +## CDB_KMeansNonspatial(subquery text, colnames text[], no_clusters int) + +K-means clustering classifies the rows of your dataset into `no_clusters` by finding the centers (means) of the variables in `colnames` and classifying each row by it's proximity to the nearest center. This method partitions space into distinct Voronoi cells. + +As a standard machine learning method, k-means clustering is an unsupervised learning technique that finds the natural clustering of values. For instance, it is useful for finding subgroups in census data leading to demographic segmentation. + +### Arguments + +| Name | Type | Description | +|------|------|-------------| +| query | TEXT | SQL query to expose the data to be used in the analysis (e.g., `SELECT * FROM iris_data`). It should contain at least the columns specified in `colnames` and the `id_colname`. | +| colnames | TEXT[] | Array of columns to be used in the analysis (e.g., `Array['petal_width', 'sepal_length', 'petal_length']`). | +| no\_clusters | INTEGER | Number of clusters for the classification of the data | +| id\_col (optional) | TEXT | The id column (default: 'cartodb_id') for identifying rows | +| standarize (optional) | BOOLEAN | Setting this to true (default) standardizes the data to have a mean at zero and a standard deviation of 1 | + +### Returns + +A table with the following columns. + +| Column | Type | Description | +|--------|------|-------------| +| cluster_label | TEXT | Label that a cluster belongs to, number from 0 to `no_clusters - 1`. | +| cluster_center | JSON | Center of the cluster that a row belongs to. The keys of the JSON object are the `colnames`, with values that are the center of the respective cluster | +| silhouettes | NUMERIC | [Silhouette score](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.silhouette_score.html#sklearn.metrics.silhouette_score) of the cluster label | +| inertia | NUMERIC | Sum of squared distances of samples to their closest cluster center | +| rowid | BIGINT | id of the original row for associating back with the original data | + +### Example Usage + +```sql +SELECT + customers.*, + km.cluster_label, + km.cluster_center, + km.silhouettes +FROM + cdb_crankshaft.CDB_KMeansNonspatial( + 'SELECT * FROM customers', + Array['customer_value', 'avg_amt_spent', 'home_median_income'], + 7) As km, + customers +WHERE + customers.cartodb_id = km.rowid +``` + +### Resources + +- Read more in [scikit-learn's documentation](http://scikit-learn.org/stable/modules/clustering.html#k-means) +- [K-means basics](https://www.datascience.com/blog/introduction-to-k-means-clustering-algorithm-learn-data-science-tutorials) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 1a01e03..b0362b8 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -1,18 +1,58 @@ -- Spatial k-means clustering -CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer, no_init integer default 20) -RETURNS table (cartodb_id integer, cluster_no integer) as $$ +CREATE OR REPLACE FUNCTION CDB_KMeans( + query TEXT, + no_clusters INTEGER, + no_init INTEGER DEFAULT 20 +) +RETURNS TABLE( + cartodb_id INTEGER, + cluster_no INTEGER +) AS $$ - from crankshaft.clustering import Kmeans - kmeans = Kmeans() - return kmeans.spatial(query, no_clusters, no_init) +from crankshaft.clustering import Kmeans +kmeans = Kmeans() +return kmeans.spatial(query, no_clusters, no_init) $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +-- Non-spatial k-means clustering +-- query: sql query to retrieve all the needed data +-- colnames: text array of column names for doing the clustering analysis +-- no_clusters: number of requested clusters +-- standardize: whether to scale variables to a mean of zero and a standard +-- deviation of 1 +-- id_colname: name of the id column -CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) -RETURNS Numeric[] AS -$$ +CREATE OR REPLACE FUNCTION CDB_KMeansNonspatial( + query TEXT, + colnames TEXT[], + no_clusters INTEGER, + standardize BOOLEAN DEFAULT true, + id_col TEXT DEFAULT 'cartodb_id' +) +RETURNS TABLE( + cluster_label text, + cluster_center json, + silhouettes numeric, + inertia numeric, + rowid bigint +) AS $$ + +from crankshaft.clustering import Kmeans +kmeans = Kmeans() +return kmeans.nonspatial(query, colnames, no_clusters, + standardize=standardize, + id_col=id_col) +$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS( + state NUMERIC[], + the_geom GEOMETRY(Point, 4326), + weight NUMERIC +) +RETURNS Numeric[] AS $$ DECLARE newX NUMERIC; newY NUMERIC; @@ -32,7 +72,8 @@ BEGIN END $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state NUMERIC[]) RETURNS GEOMETRY AS $$ BEGIN diff --git a/src/pg/test/expected/11_kmeans_test.out b/src/pg/test/expected/11_kmeans_test.out index 8c6ffa1..85b8b13 100644 --- a/src/pg/test/expected/11_kmeans_test.out +++ b/src/pg/test/expected/11_kmeans_test.out @@ -1,10 +1,43 @@ \pset format unaligned \set ECHO all -SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); +-- spatial kmeans +SELECT + count(DISTINCT cluster_no) as clusters +FROM + cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); clusters 2 (1 row) -SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; +-- weighted mean +SELECT + count(*) clusters +FROM ( + SELECT + cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), + code + FROM ppoints + GROUP BY code +) p; clusters 52 (1 row) +-- nonspatial kmeans +SELECT + cluster_label::int in (0, 1) As cluster_label, + cluster_center::json->>'col1' As cc_col1, + cluster_center::json->>'col2' As cc_col2, + silhouettes, + inertia, + rowid +FROM cdb_crankshaft.CDB_KMeansNonspatial( + 'SELECT unnest(Array[1, 1, 10, 10]) As col1, ' || + 'unnest(Array[100, 100, 2, 2]) As col2, ' || + 'unnest(Array[1, 2, 3, 4]) As cartodb_id ', + Array['col1', 'col2']::text[], + 2); +cluster_label|cc_col1|cc_col2|silhouettes|inertia|rowid +t|-1.0|1.0|1.0|0.0|1 +t|-1.0|1.0|1.0|0.0|2 +t|1.0|-1.0|1.0|0.0|3 +t|1.0|-1.0|1.0|0.0|4 +(4 rows) diff --git a/src/pg/test/sql/11_kmeans_test.sql b/src/pg/test/sql/11_kmeans_test.sql index 2298b85..a44e315 100644 --- a/src/pg/test/sql/11_kmeans_test.sql +++ b/src/pg/test/sql/11_kmeans_test.sql @@ -1,6 +1,34 @@ \pset format unaligned \set ECHO all -SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); +-- spatial kmeans +SELECT + count(DISTINCT cluster_no) as clusters +FROM + cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); -SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; +-- weighted mean +SELECT + count(*) clusters +FROM ( + SELECT + cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), + code + FROM ppoints + GROUP BY code +) p; + +-- nonspatial kmeans +SELECT + cluster_label::int in (0, 1) As cluster_label, + cluster_center::json->>'col1' As cc_col1, + cluster_center::json->>'col2' As cc_col2, + silhouettes, + inertia, + rowid +FROM cdb_crankshaft.CDB_KMeansNonspatial( + 'SELECT unnest(Array[1, 1, 10, 10]) As col1, ' || + 'unnest(Array[100, 100, 2, 2]) As col2, ' || + 'unnest(Array[1, 2, 3, 4]) As cartodb_id ', + Array['col1', 'col2']::text[], + 2); diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 12737bf..3d5225a 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -44,8 +44,32 @@ class AnalysisDataProvider(object): return plpy.execute(query) @verify_data - def get_nonspatial_kmeans(self, query): - """fetch data for non-spatial kmeans""" + def get_nonspatial_kmeans(self, params): + """ + Fetch data for non-spatial k-means. + + Inputs - a dict (params) with the following keys: + colnames: a (text) list of column names (e.g., + `['andy', 'cookie']`) + id_col: the name of the id column (e.g., `'cartodb_id'`) + subquery: the subquery for exposing the data (e.g., + SELECT * FROM favorite_things) + Output: + A SQL query for packaging the data for consumption within + `KMeans().nonspatial`. Format will be a list of length one, + with the first element a dict with keys ('rowid', 'attr1', + 'attr2', ...) + """ + agg_cols = ', '.join([ + 'array_agg({0}) As arr_col{1}'.format(val, idx+1) + for idx, val in enumerate(params['colnames']) + ]) + query = ''' + SELECT {cols}, array_agg({id_col}) As rowid + FROM ({subquery}) As a + '''.format(subquery=params['subquery'], + id_col=params['id_col'], + cols=agg_cols).strip() return plpy.execute(query) @verify_data diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py index 094d47b..6d22d44 100644 --- a/src/py/crankshaft/crankshaft/clustering/kmeans.py +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -30,3 +30,84 @@ class Kmeans(object): km = KMeans(n_clusters=no_clusters, n_init=no_init) labels = km.fit_predict(zip(xs, ys)) return zip(ids, labels) + + def nonspatial(self, subquery, colnames, no_clusters=5, + standardize=True, id_col='cartodb_id'): + """ + Arguments: + query (string): A SQL query to retrieve the data required to do the + k-means clustering analysis, like so: + SELECT * FROM iris_flower_data + colnames (list): a list of the column names which contain the data + of interest, like so: ['sepal_width', + 'petal_width', + 'sepal_length', + 'petal_length'] + no_clusters (int): number of clusters (greater than zero) + id_col (string): name of the input id_column + + Returns: + A list of tuples with the following columns: + cluster labels: a label for the cluster that the row belongs to + centers: center of the cluster that this row belongs to + silhouettes: silhouette measure for this value + rowid: row that these values belong to (corresponds to the value in + `id_col`) + """ + import json + from sklearn import metrics + + params = { + "colnames": colnames, + "subquery": subquery, + "id_col": id_col + } + + data = self.data_provider.get_nonspatial_kmeans(params) + + # fill array with values for k-means clustering + if standardize: + cluster_columns = _scale_data( + _extract_columns(data)) + else: + cluster_columns = _extract_columns(data) + + kmeans = KMeans(n_clusters=no_clusters, + random_state=0).fit(cluster_columns) + + centers = [json.dumps(dict(zip(colnames, c))) + for c in kmeans.cluster_centers_[kmeans.labels_]] + + silhouettes = metrics.silhouette_samples(cluster_columns, + kmeans.labels_, + metric='sqeuclidean') + + return zip(kmeans.labels_, + centers, + silhouettes, + [kmeans.inertia_] * kmeans.labels_.shape[0], + data[0]['rowid']) + + +# -- Preprocessing steps + +def _extract_columns(data): + """ + Extract the features from the query and pack them into a NumPy array + data (list of dicts): result of the kmeans request + """ + # number of columns minus rowid column + n_cols = len(data[0]) - 1 + return np.array([data[0]['arr_col{0}'.format(i+1)] + for i in xrange(n_cols)], + dtype=float).T + + +def _scale_data(features): + """ + Scale all input columns to center on 0 with a standard devation of 1 + features (numpy matrix): features of dimension (n_features, n_samples) + """ + from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() + return scaler.fit_transform(features) diff --git a/src/py/crankshaft/test/test_clustering_kmeans.py b/src/py/crankshaft/test/test_clustering_kmeans.py index 93633b0..c118d34 100644 --- a/src/py/crankshaft/test/test_clustering_kmeans.py +++ b/src/py/crankshaft/test/test_clustering_kmeans.py @@ -2,17 +2,12 @@ import unittest import numpy as np -# from mock_plpy import MockPlPy -# plpy = MockPlPy() -# -# import sys -# sys.modules['plpy'] = plpy from helper import fixture_file from crankshaft.clustering import Kmeans from crankshaft.analysis_data_provider import AnalysisDataProvider import crankshaft.clustering as cc - from crankshaft import random_seeds + import json from collections import OrderedDict @@ -24,7 +19,7 @@ class FakeDataProvider(AnalysisDataProvider): def get_spatial_kmeans(self, query): return self.mocked_result - def get_nonspatial_kmeans(self, query, standarize): + def get_nonspatial_kmeans(self, query): return self.mocked_result @@ -54,3 +49,39 @@ class KMeansTest(unittest.TestCase): self.assertEqual(len(np.unique(labels)), 2) self.assertEqual(len(c1), 20) self.assertEqual(len(c2), 20) + + +class KMeansNonspatialTest(unittest.TestCase): + """Testing class for k-means non-spatial""" + + def setUp(self): + self.params = {"subquery": "SELECT * FROM TABLE", + "n_clusters": 5} + + def test_kmeans_nonspatial(self): + """ + test for k-means non-spatial + """ + # data from: + # http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn-cluster-kmeans + data_raw = [OrderedDict([("arr_col1", [1, 1, 1, 4, 4, 4]), + ("arr_col2", [2, 4, 0, 2, 4, 0]), + ("rowid", [1, 2, 3, 4, 5, 6])])] + + random_seeds.set_random_seeds(1234) + kmeans = Kmeans(FakeDataProvider(data_raw)) + clusters = kmeans.nonspatial('subquery', ['col1', 'col2'], 2) + + cl1 = clusters[0][0] + cl2 = clusters[3][0] + + for idx, val in enumerate(clusters): + if idx < 3: + self.assertEqual(val[0], cl1) + else: + self.assertEqual(val[0], cl2) + + # raises exception for no data + with self.assertRaises(Exception): + kmeans = Kmeans(FakeDataProvider([])) + kmeans.nonspatial('subquery', ['col1', 'col2'], 2)