diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 3d3dcdc..8e060f4 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -57,9 +57,9 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( query TEXT, - variable_name TEXT, - target_query TEXT, + variable TEXT, feature_columns TEXT[], + target_query TEXT, n_estimators INTEGER DEFAULT 1200, max_depth INTEGER DEFAULT 3, subsample DOUBLE PRECISION DEFAULT 0.5, @@ -70,5 +70,5 @@ AS $$ from crankshaft.segmentation import Segmentation seg = Segmentation() model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} - return seg.create_and_predict_segment(query, variable_name, feature_columns, target_query, model_params) + return seg.create_and_predict_segment(query, variable, feature_columns, target_query, model_params) $$ LANGUAGE plpythonu; diff --git a/src/py/crankshaft/crankshaft/analysis_data_provider.py b/src/py/crankshaft/crankshaft/analysis_data_provider.py index 9d373d9..9d0d468 100644 --- a/src/py/crankshaft/crankshaft/analysis_data_provider.py +++ b/src/py/crankshaft/crankshaft/analysis_data_provider.py @@ -77,6 +77,7 @@ class AnalysisDataProvider(object): "target": variable, "features": feature_columns} """ + plpy.notice("featurecols: {}".format(str(params))) columns = ', '.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in params['features']]) @@ -88,6 +89,7 @@ class AnalysisDataProvider(object): '''.format(subquery=params['subquery'], target=params['target'], columns=columns) + plpy.notice("Query: {}".format(query)) try: data = plpy.execute(query) return data diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 83a4ea8..b5f3654 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -70,7 +70,7 @@ class Segmentation(object): params = {"subquery": target_query, "id_col": id_col} - target, features, target_mean, feature_means = self.clean_data(variable, feature_columns, query) + target, features, target_mean, feature_means = self.clean_data(query, variable, feature_columns) model, accuracy = train_model(target, features, model_params, 0.2) result = self.predict_segment(model, feature_columns, target_query, @@ -83,7 +83,7 @@ class Segmentation(object): rowid = [{'ids': [2.9, 4.9, 4, 5, 6]}] ''' - return zip(rowid, result, accuracy_array) + return zip(rowid[0]['ids'], result, accuracy_array) def predict_segment(self, model, feature_columns, target_query, feature_means): @@ -104,6 +104,9 @@ class Segmentation(object): results = [] cursors = self.data_provider.get_segmentation_predict_data(params) + import plpy + plpy.notice("cursor:{}".format(cursors)) + ''' cursors = [{'features': [[m1[0],m2[0],m3[0]],[m1[1],m2[1],m3[1]], [m1[2],m2[2],m3[2]]]}] @@ -113,12 +116,14 @@ class Segmentation(object): rows = cursors.fetch(batch_size) if not rows: break - batch = np.row_stack([np.array(row['features'], dtype=float) - for row in rows]) + batch = np.row_stack([np.array(row['features']) + for row in rows]).astype(float) # Need to fix this to global mean. This will cause weird effects - batch = replace_nan_with_mean(batch, feature_means) + batch = replace_nan_with_mean(batch, feature_means)[0] + import plpy + plpy.notice("BATCH: {}".format(batch)) prediction = model.predict(batch) results.append(prediction) @@ -136,17 +141,16 @@ class Segmentation(object): data = self.data_provider.get_segmentation_model_data(params) ''' - data = [{'target': [2.9, 4.9, 4, 5, 6]}, - {'feature1': [1,2,3,4]}, {'feature2' : [2,3,4,5]} - ] + data = [{'target': [2.9, 4.9, 4, 5, 6], + 'feature1': [1,2,3,4], 'feature2' : [2,3,4,5]}] ''' # extract target data from plpy object - target = np.array(data[0]['target']) + target = np.array(data[0]['target'], dtype=float) # put n feature data arrays into an n x m array of arrays - features = np.column_stack([np.array(data[0][col], dtype=float) - for col in feature_columns]) + features = np.column_stack([np.array(data[0][col]) + for col in feature_columns]).astype(float) features, feature_means = replace_nan_with_mean(features) target, target_mean = replace_nan_with_mean(target) @@ -164,11 +168,28 @@ def replace_nan_with_mean(array, means=None): # TODO: update code to take in avgs parameter # returns an array of rows and column indices - indices = np.where(np.isnan(array)) + # import plpy + # plpy.notice("array is of type: {}".format(type(array))) + # plpy.notice("ARRAY: {}".format(array)) + nanvals = np.isnan(array) + indices = np.where(nanvals) - if not means: - for col in np.shape(array)[1]: - means[col] = np.mean(array[~np.isnan(array[:, col]), col]) + if means is None: + means = {} + + def loops(array, axis): + try: + return np.shape(array)[axis] + except IndexError: + return 1 + + ran = loops(array, 1) + if ran == 1: + array = np.array(array) + means[0] = np.mean(array[~np.isnan(array)]) + else: + for col in range(ran): + means[col] = np.mean(array[~np.isnan(array[:, col]), col]) # iterate through entries which have nan values for row, col in zip(*indices): diff --git a/src/py/crankshaft/test/test_segmentation.py b/src/py/crankshaft/test/test_segmentation.py index faeafef..28157d2 100644 --- a/src/py/crankshaft/test/test_segmentation.py +++ b/src/py/crankshaft/test/test_segmentation.py @@ -87,7 +87,7 @@ class SegmentationTest(unittest.TestCase): 'subsample': 0.5, 'learning_rate': 0.01, 'min_samples_leaf': 1} - data = [{'query': + data = [{'query': 'select * FROM research_team', 'target': [], 'x1': [], 'x2': [], @@ -111,7 +111,7 @@ class SegmentationTest(unittest.TestCase): # Before here figure out how to set up the data provider # After use data prodiver to run the query and test results. - seg = Segmentation(data_provider=) + seg = Segmentation(RawDataProvider([])) result = seg.create_and_predict_segment( 'select * from training',