diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 3979ca4..67b27ec 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -1,3 +1,41 @@ + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1 + ) +RETURNS TABLE( cartodb_id Numeric, prediction Numeric , accuracy Numeric) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = { 'n_estimators' : n_estimators, + 'max_depth' : max_depth, + 'subsample' : subsample, + 'learning_rate' : learning_rate, + 'min_samples_leaf' : min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg( np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ Language plpythonu; + CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( query TEXT, @@ -16,4 +54,3 @@ AS $$ model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} return create_and_predict_segment(query,variable_name,target_table, model_params) $$ LANGUAGE plpythonu; - diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 9ba4c7c..4dbcd69 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -29,6 +29,18 @@ def get_data(variable, feature_columns, query): return replace_nan_with_mean(target), replace_nan_with_mean(features) + +def create_and_predict_segment_agg(target, features, target_features, target_ids,model_parameters): + clean_target = replace_nan_with_mean(target) + clean_features = replace_nan_with_mean(features) + target_features = replace_nan_with_mean(target_features) + + model, accuracy = train_model(clean_target,clean_features, model_parameters, 0.2) + prediction = model.predict(target_features) + return zip(target_ids, prediction, np.full(prediction.shape, accuracy)) + + + def create_and_predict_segment(query,variable,target_query,model_params): """ generate a segment with machine learning @@ -48,9 +60,7 @@ def create_and_predict_segment(query,variable,target_query,model_params): def train_model(target,features,model_params,test_split): features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) model = GradientBoostingRegressor(**model_params) - plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) - plpy.notice('model trained') accuracy = calculate_model_accuracy(model,features,target) return model, accuracy @@ -82,10 +92,8 @@ def predict_segment(model,features,target_query): #Need to fix this. Should be global mean. This will cause weird effects batch = replace_nan_with_mean(batch) - plpy.notice(len(batch)) prediction = model.predict(batch) results.append(prediction) - plpy.notice('predicting: predicted') cartodb_ids = plpy.execute('select array_agg(cartodb_id order by cartodb_id) as cartodb_ids from ({0}) a '.format(target_query))[0]['cartodb_ids']