From 746dcc97236f3ebc626e0bcd3be298f583a14f97 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Sat, 5 Mar 2016 17:55:32 -0500 Subject: [PATCH 001/183] segmentation --- pg/sql/0.0.1/05_segmentation.sql | 27 ++++ python/crankshaft/crankshaft/__init__.py | 1 + .../crankshaft/segmentation/__init__.py | 0 .../crankshaft/segmentation/segmentation.py | 118 ++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 pg/sql/0.0.1/05_segmentation.sql create mode 100644 python/crankshaft/crankshaft/segmentation/__init__.py create mode 100644 python/crankshaft/crankshaft/segmentation/segmentation.py diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql new file mode 100644 index 0000000..8e23ad6 --- /dev/null +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -0,0 +1,27 @@ +CREATE OR REPLACE FUNCTION + cdb_create_segment ( + segment_name TEXT, + table_name TEXT, + column_name TEXT, + geoid_column TEXT DEFAULT 'geoid', + census_table TEXT DEFAULT 'block_groups' + ) +RETURNS NUMERIC +AS $$ + from crankshaft.segmentation import create_segemnt + # TODO: use named parameters or a dictionary + return create_segment('table') +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + cdb_predict_segment ( + segment_name TEXT, + geoid_column TEXT DEFAULT 'geoid', + census_table TEXT DEFAULT 'block_groups' + ) +RETURNS TABLE(geoid TEXT, prediction NUMERIC) +AS $$ + from crankshaft.segmentation import create_segemnt + # TODO: use named parameters or a dictionary + return create_segment('table') +$$ LANGUAGE plpythonu; diff --git a/python/crankshaft/crankshaft/__init__.py b/python/crankshaft/crankshaft/__init__.py index d07e330..bc8e065 100644 --- a/python/crankshaft/crankshaft/__init__.py +++ b/python/crankshaft/crankshaft/__init__.py @@ -1,2 +1,3 @@ import random_seeds import clustering +import segmentation diff --git a/python/crankshaft/crankshaft/segmentation/__init__.py b/python/crankshaft/crankshaft/segmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py new file mode 100644 index 0000000..3894122 --- /dev/null +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -0,0 +1,118 @@ +""" +Segmentation creation and prediction +""" + +import sklearn +import numpy as np +import pandas as pd +import pickle +import plpy +from sklearn.ensemble import ExtraTreesRegressor +from sklearn import metrics +from sklearn.cross_validation import train_test_split + +# High level interface --------------------------------------- + +def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): + """ + generate a segment with machine learning + Stuart Lynn + """ + data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) + features = data[data.columns.difference([column_name, 'geoid'])] + target, mean, std = normalize(data[column_name]) + model, accuracy = train_model(target,features, test_split=0.2) + save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + return accuracy + +def normalize(target): + mean = np.mean(target) + std = no.std(target) + return (target - mean)/std, mean, std + +def denormalize(target, mean ,std): + return target*std + mean + +def train_model(target,features,test_split): + features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) + model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) + model.fit(features_train, target_train) + accuracy = calculate_model_accuracy(model,features,target) + return model, accuracy + +def calculate_model_accuracy(model,features,target): + prediction = self.model.predict(features) + return metrics.mean_squared_error(prediction,target)/np.std(target) + +def join_with_census(table_name, column_name, geoid_column, census_table): + coulmns = plpy.execute('select {census_table}.* limit 1 ') + feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) + join_data = plpy.execute(''' + WITH region_extent AS ( + SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; + ) + SELECT {features_names}, {table_name}.{column_name} + FROM {table_name} ,region_extent + JOIN {census_table} + ON {table_name}.{geoid_column} = {census_table}.geoid + WHERE {census_table}.the_geom && region_extent.table_extent + '''.format(**locals())) + + if len(join_data) == 0: + plpy.notice('Failed to join with census data') + + return join_data + +def cdb_predict_segment(segment_name,geoid_column,census_table): + """ + predict a segment with machine learning + Stuart Lynn + """ + data = fetch_model(segment_name) + model = data['model'] + features = ",".join(data['features']) + targets = plpy.execute('select {features} from {census_table}') + geo_ids = plpy.execute('select geoid from {census_table}') + result = model.predict(targets) + return zip(geo_ids,prediction) + + +def fetch_model(model_name): + """ + fetch a model from storage + """ + data = plpy.execute('select * from models where name={model_name}') + if len(data)==0: + plpy.notice('model not found') + data = data[0] + data['model'] = pickle.load(data['model']) + return data + + +def create_model_table(model_name): + """ + create the model table if requred + """ + plpy.execute(''' + CREATE table IF NOT EXISTS _cdb_models( + name TEXT, + model BLOB, + features TEXT[], + accuracy NUMERIC, + table_name TEXT, + )''') + +def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): + """ + save a model to the model table for later use + """ + + plpy.execute(''' + DELETE FROM _cdb_models WHERE model_name = {model_name} + '''.format(**locals())) + + plpy.execute(""" + INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) + """) + +def From d96d6b2c482fe71472a1606cb0a45379003903da Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 7 Mar 2016 11:41:37 -0500 Subject: [PATCH 002/183] fleshing out segmentation --- .../crankshaft/segmentation/__init__.py | 1 + .../crankshaft/segmentation/segmentation.py | 236 +++++++++--------- 2 files changed, 119 insertions(+), 118 deletions(-) diff --git a/python/crankshaft/crankshaft/segmentation/__init__.py b/python/crankshaft/crankshaft/segmentation/__init__.py index e69de29..b825e85 100644 --- a/python/crankshaft/crankshaft/segmentation/__init__.py +++ b/python/crankshaft/crankshaft/segmentation/__init__.py @@ -0,0 +1 @@ +from segmentation import * diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index 3894122..4f99573 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -1,118 +1,118 @@ -""" -Segmentation creation and prediction -""" - -import sklearn -import numpy as np -import pandas as pd -import pickle -import plpy -from sklearn.ensemble import ExtraTreesRegressor -from sklearn import metrics -from sklearn.cross_validation import train_test_split - -# High level interface --------------------------------------- - -def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): - """ - generate a segment with machine learning - Stuart Lynn - """ - data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) - features = data[data.columns.difference([column_name, 'geoid'])] - target, mean, std = normalize(data[column_name]) - model, accuracy = train_model(target,features, test_split=0.2) - save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) - return accuracy - -def normalize(target): - mean = np.mean(target) - std = no.std(target) - return (target - mean)/std, mean, std - -def denormalize(target, mean ,std): - return target*std + mean - -def train_model(target,features,test_split): - features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) - model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) - model.fit(features_train, target_train) - accuracy = calculate_model_accuracy(model,features,target) - return model, accuracy - -def calculate_model_accuracy(model,features,target): - prediction = self.model.predict(features) - return metrics.mean_squared_error(prediction,target)/np.std(target) - -def join_with_census(table_name, column_name, geoid_column, census_table): - coulmns = plpy.execute('select {census_table}.* limit 1 ') - feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) - join_data = plpy.execute(''' - WITH region_extent AS ( - SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; - ) - SELECT {features_names}, {table_name}.{column_name} - FROM {table_name} ,region_extent - JOIN {census_table} - ON {table_name}.{geoid_column} = {census_table}.geoid - WHERE {census_table}.the_geom && region_extent.table_extent - '''.format(**locals())) - - if len(join_data) == 0: - plpy.notice('Failed to join with census data') - - return join_data - -def cdb_predict_segment(segment_name,geoid_column,census_table): - """ - predict a segment with machine learning - Stuart Lynn - """ - data = fetch_model(segment_name) - model = data['model'] - features = ",".join(data['features']) - targets = plpy.execute('select {features} from {census_table}') - geo_ids = plpy.execute('select geoid from {census_table}') - result = model.predict(targets) - return zip(geo_ids,prediction) - - -def fetch_model(model_name): - """ - fetch a model from storage - """ - data = plpy.execute('select * from models where name={model_name}') - if len(data)==0: - plpy.notice('model not found') - data = data[0] - data['model'] = pickle.load(data['model']) - return data - - -def create_model_table(model_name): - """ - create the model table if requred - """ - plpy.execute(''' - CREATE table IF NOT EXISTS _cdb_models( - name TEXT, - model BLOB, - features TEXT[], - accuracy NUMERIC, - table_name TEXT, - )''') - -def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): - """ - save a model to the model table for later use - """ - - plpy.execute(''' - DELETE FROM _cdb_models WHERE model_name = {model_name} - '''.format(**locals())) - - plpy.execute(""" - INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) - """) - -def +# """ +# Segmentation creation and prediction +# """ +# +# import sklearn +# import numpy as np +# import pandas as pd +# import pickle +# import plpy +# from sklearn.ensemble import ExtraTreesRegressor +# from sklearn import metrics +# from sklearn.cross_validation import train_test_split +# +# # High level interface --------------------------------------- +# +# def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): +# """ +# generate a segment with machine learning +# Stuart Lynn +# """ +# data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) +# features = data[data.columns.difference([column_name, 'geoid'])] +# target, mean, std = normalize(data[column_name]) +# model, accuracy = train_model(target,features, test_split=0.2) +# save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) +# return accuracy +# +# def normalize(target): +# mean = np.mean(target) +# std = no.std(target) +# return (target - mean)/std, mean, std +# +# def denormalize(target, mean ,std): +# return target*std + mean +# +# def train_model(target,features,test_split): +# features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) +# model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) +# model.fit(features_train, target_train) +# accuracy = calculate_model_accuracy(model,features,target) +# return model, accuracy +# +# def calculate_model_accuracy(model,features,target): +# prediction = self.model.predict(features) +# return metrics.mean_squared_error(prediction,target)/np.std(target) +# +# def join_with_census(table_name, column_name, geoid_column, census_table): +# coulmns = plpy.execute('select {census_table}.* limit 1 ') +# feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) +# join_data = plpy.execute(''' +# WITH region_extent AS ( +# SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; +# ) +# SELECT {features_names}, {table_name}.{column_name} +# FROM {table_name} ,region_extent +# JOIN {census_table} +# ON {table_name}.{geoid_column} = {census_table}.geoid +# WHERE {census_table}.the_geom && region_extent.table_extent +# '''.format(**locals())) +# +# if len(join_data) == 0: +# plpy.notice('Failed to join with census data') +# +# return join_data +# +# def cdb_predict_segment(segment_name,geoid_column,census_table): +# """ +# predict a segment with machine learning +# Stuart Lynn +# """ +# data = fetch_model(segment_name) +# model = data['model'] +# features = ",".join(data['features']) +# targets = plpy.execute('select {features} from {census_table}') +# geo_ids = plpy.execute('select geoid from {census_table}') +# result = model.predict(targets) +# return zip(geo_ids,prediction) +# +# +# def fetch_model(model_name): +# """ +# fetch a model from storage +# """ +# data = plpy.execute('select * from models where name={model_name}') +# if len(data)==0: +# plpy.notice('model not found') +# data = data[0] +# data['model'] = pickle.load(data['model']) +# return data +# +# +# def create_model_table(model_name): +# """ +# create the model table if requred +# """ +# plpy.execute(''' +# CREATE table IF NOT EXISTS _cdb_models( +# name TEXT, +# model BLOB, +# features TEXT[], +# accuracy NUMERIC, +# table_name TEXT, +# )''') +# +# def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): +# """ +# save a model to the model table for later use +# """ +# +# plpy.execute(''' +# DELETE FROM _cdb_models WHERE model_name = {model_name} +# '''.format(**locals())) +# +# plpy.execute(""" +# INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) +# """) +# +# def From f885cc9f7b317600fa9b303fa35a5eed4fbe81a4 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 9 Mar 2016 20:04:12 -0500 Subject: [PATCH 003/183] more segmentation fleshing out --- .../crankshaft/segmentation/segmentation.py | 236 +++++++++--------- 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index 4f99573..acdae51 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -1,118 +1,118 @@ -# """ -# Segmentation creation and prediction -# """ -# -# import sklearn -# import numpy as np -# import pandas as pd -# import pickle -# import plpy -# from sklearn.ensemble import ExtraTreesRegressor -# from sklearn import metrics -# from sklearn.cross_validation import train_test_split -# -# # High level interface --------------------------------------- -# -# def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): -# """ -# generate a segment with machine learning -# Stuart Lynn -# """ -# data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) -# features = data[data.columns.difference([column_name, 'geoid'])] -# target, mean, std = normalize(data[column_name]) -# model, accuracy = train_model(target,features, test_split=0.2) -# save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) -# return accuracy -# -# def normalize(target): -# mean = np.mean(target) -# std = no.std(target) -# return (target - mean)/std, mean, std -# -# def denormalize(target, mean ,std): -# return target*std + mean -# -# def train_model(target,features,test_split): -# features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) -# model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) -# model.fit(features_train, target_train) -# accuracy = calculate_model_accuracy(model,features,target) -# return model, accuracy -# -# def calculate_model_accuracy(model,features,target): -# prediction = self.model.predict(features) -# return metrics.mean_squared_error(prediction,target)/np.std(target) -# -# def join_with_census(table_name, column_name, geoid_column, census_table): -# coulmns = plpy.execute('select {census_table}.* limit 1 ') -# feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) -# join_data = plpy.execute(''' -# WITH region_extent AS ( -# SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; -# ) -# SELECT {features_names}, {table_name}.{column_name} -# FROM {table_name} ,region_extent -# JOIN {census_table} -# ON {table_name}.{geoid_column} = {census_table}.geoid -# WHERE {census_table}.the_geom && region_extent.table_extent -# '''.format(**locals())) -# -# if len(join_data) == 0: -# plpy.notice('Failed to join with census data') -# -# return join_data -# -# def cdb_predict_segment(segment_name,geoid_column,census_table): -# """ -# predict a segment with machine learning -# Stuart Lynn -# """ -# data = fetch_model(segment_name) -# model = data['model'] -# features = ",".join(data['features']) -# targets = plpy.execute('select {features} from {census_table}') -# geo_ids = plpy.execute('select geoid from {census_table}') -# result = model.predict(targets) -# return zip(geo_ids,prediction) -# -# -# def fetch_model(model_name): -# """ -# fetch a model from storage -# """ -# data = plpy.execute('select * from models where name={model_name}') -# if len(data)==0: -# plpy.notice('model not found') -# data = data[0] -# data['model'] = pickle.load(data['model']) -# return data -# -# -# def create_model_table(model_name): -# """ -# create the model table if requred -# """ -# plpy.execute(''' -# CREATE table IF NOT EXISTS _cdb_models( -# name TEXT, -# model BLOB, -# features TEXT[], -# accuracy NUMERIC, -# table_name TEXT, -# )''') -# -# def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): -# """ -# save a model to the model table for later use -# """ -# -# plpy.execute(''' -# DELETE FROM _cdb_models WHERE model_name = {model_name} -# '''.format(**locals())) -# -# plpy.execute(""" -# INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) -# """) -# -# def +""" +Segmentation creation and prediction +""" + +import sklearn +import numpy as np +import pandas as pd +import pickle +import plpy +from sklearn.ensemble import ExtraTreesRegressor +from sklearn import metrics +from sklearn.cross_validation import train_test_split + +# High level interface --------------------------------------- + +def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): + """ + generate a segment with machine learning + Stuart Lynn + """ + data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table,)) + features = data[data.columns.difference([column_name, 'geoid'])] + target, mean, std = normalize(data[column_name]) + model, accuracy = train_model(target,features, test_split=0.2) + save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + return accuracy + +def normalize(target): + mean = np.mean(target) + std = no.std(target) + return (target - mean)/std, mean, std + +def denormalize(target, mean ,std): + return target*std + mean + +def train_model(target,features,test_split): + features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) + model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) + model.fit(features_train, target_train) + accuracy = calculate_model_accuracy(model,features,target) + return model, accuracy + +def calculate_model_accuracy(model,features,target): + prediction = self.model.predict(features) + return metrics.mean_squared_error(prediction,target)/np.std(target) + +def join_with_census(table_name, column_name, geoid_column, census_table): + coulmns = plpy.execute('select {census_table}.* limit 1 ') + feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) + join_data = plpy.execute(''' + WITH region_extent AS ( + SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; + ) + SELECT {features_names}, {table_name}.{column_name} + FROM {table_name} ,region_extent + JOIN {census_table} + ON {table_name}.{geoid_column} = {census_table}.geoid + WHERE {census_table}.the_geom && region_extent.table_extent + '''.format(**locals())) + + if len(join_data) == 0: + plpy.notice('Failed to join with census data') + + return join_data + +def cdb_predict_segment(segment_name,geoid_column,census_table): + """ + predict a segment with machine learning + Stuart Lynn + """ + data = fetch_model(segment_name) + model = data['model'] + features = ",".join(data['features']) + targets = plpy.execute('select {features} from {census_table}') + geo_ids = plpy.execute('select geoid from {census_table}') + result = model.predict(targets) + return zip(geo_ids,prediction) + + +def fetch_model(model_name): + """ + fetch a model from storage + """ + data = plpy.execute('select * from models where name={model_name}') + if len(data)==0: + plpy.notice('model not found') + data = data[0] + data['model'] = pickle.load(data['model']) + return data + + +def create_model_table(model_name): + """ + create the model table if requred + """ + plpy.execute(''' + CREATE table IF NOT EXISTS _cdb_models( + name TEXT, + model BLOB, + features TEXT[], + accuracy NUMERIC, + table_name TEXT, + )''') + +def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): + """ + save a model to the model table for later use + """ + + plpy.execute(''' + DELETE FROM _cdb_models WHERE model_name = {model_name} + '''.format(**locals())) + + plpy.execute(""" + INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) + """) + +def From fcf57289fc6bb027374632b8ffe9909ff6cc01c8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 10 Mar 2016 12:50:50 -0500 Subject: [PATCH 004/183] Training section now works --- pg/crankshaft--0.0.1.sql | 27 ++++++++ pg/sql/0.0.1/05_segmentation.sql | 4 +- .../crankshaft/segmentation/segmentation.py | 65 ++++++++++++------- python/crankshaft/setup.py | 3 +- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/pg/crankshaft--0.0.1.sql b/pg/crankshaft--0.0.1.sql index 436beea..c72e5fc 100644 --- a/pg/crankshaft--0.0.1.sql +++ b/pg/crankshaft--0.0.1.sql @@ -137,6 +137,33 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE; +CREATE OR REPLACE FUNCTION + cdb_create_segment ( + segment_name TEXT, + table_name TEXT, + column_name TEXT, + geoid_column TEXT DEFAULT 'geoid', + census_table TEXT DEFAULT 'block_groups' + ) +RETURNS NUMERIC +AS $$ + from crankshaft import segmentation + # TODO: use named parameters or a dictionary + return segmentation.create_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + cdb_predict_segment ( + segment_name TEXT, + geoid_column TEXT DEFAULT 'geoid', + census_table TEXT DEFAULT 'block_groups' + ) +RETURNS TABLE(geoid TEXT, prediction NUMERIC) +AS $$ + from crankshaft.segmentation import create_segemnt + # TODO: use named parameters or a dictionary + return create_segment('table') +$$ LANGUAGE plpythonu; -- Make sure by default there are no permissions for publicuser -- NOTE: this happens at extension creation time, as part of an implicit transaction. -- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql index 8e23ad6..cce29b6 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -8,9 +8,9 @@ CREATE OR REPLACE FUNCTION ) RETURNS NUMERIC AS $$ - from crankshaft.segmentation import create_segemnt + from crankshaft import segmentation # TODO: use named parameters or a dictionary - return create_segment('table') + return segmentation.create_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index acdae51..201e7e4 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -13,57 +13,71 @@ from sklearn.cross_validation import train_test_split # High level interface --------------------------------------- -def cdb_create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): +def create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): """ generate a segment with machine learning Stuart Lynn """ - data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table,)) - features = data[data.columns.difference([column_name, 'geoid'])] + data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) + features = data[data.columns.difference([column_name, 'geoid','the_geom'])] target, mean, std = normalize(data[column_name]) model, accuracy = train_model(target,features, test_split=0.2) - save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + # predict_segment return accuracy def normalize(target): mean = np.mean(target) - std = no.std(target) + std = np.std(target) + plpy.notice('mean '+str(mean)+" std : "+str(std)) return (target - mean)/std, mean, std def denormalize(target, mean ,std): return target*std + mean def train_model(target,features,test_split): + plpy.notice('training the model') + plpy.notice('dataframe shape '+ str(np.shape(features))) + plpy.notice('dataframe columns '+ str(features.dtypes)) + features = features.dropna(axis =1, how='all').fillna(0) + target = target.fillna(0) features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) + plpy.notice('training the model test train split') model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) + plpy.notice('training the model created tree') + plpy.notice('features '+str(np.shape(features_train))+" "+str(np.shape(features_test)) ) + model.fit(features_train, target_train) + plpy.notice('training the model fitting model') accuracy = calculate_model_accuracy(model,features,target) return model, accuracy def calculate_model_accuracy(model,features,target): - prediction = self.model.predict(features) + prediction = model.predict(features) return metrics.mean_squared_error(prediction,target)/np.std(target) def join_with_census(table_name, column_name, geoid_column, census_table): - coulmns = plpy.execute('select {census_table}.* limit 1 ') - feature_names = ",".join(columns.keys.difference(['the_geom','cartodb_id'])) + columns = plpy.execute('select * from {census_table} limit 1 '.format(**locals())) + combined_columns = [ a for a in columns[0].keys() if a not in ['the_geom','cartodb_id','geoid']] + feature_names = ",".join([ " {census_table}.\"{a}\" as \"{a}\" ".format(**locals()) for a in combined_columns]) + plpy.notice('joining with census data') join_data = plpy.execute(''' - WITH region_extent AS ( - SELECT ST_Extent(the_geom) as table_extent FROM {table_name}; - ) - SELECT {features_names}, {table_name}.{column_name} - FROM {table_name} ,region_extent + + SELECT {feature_names}, {table_name}.{column_name} + FROM {table_name} JOIN {census_table} - ON {table_name}.{geoid_column} = {census_table}.geoid - WHERE {census_table}.the_geom && region_extent.table_extent + ON {table_name}.{geoid_column}::numeric = {census_table}.geoid::numeric '''.format(**locals())) if len(join_data) == 0: plpy.notice('Failed to join with census data') - return join_data + return query_to_dictionary(join_data) -def cdb_predict_segment(segment_name,geoid_column,census_table): +def query_to_dictionary(result): + return [ dict(zip(r.keys(), r.values())) for r in result ] + +def predict_segment(model,features,geoid_column,census_table): """ predict a segment with machine learning Stuart Lynn @@ -89,30 +103,31 @@ def fetch_model(model_name): return data -def create_model_table(model_name): +def create_model_table(): """ create the model table if requred """ plpy.execute(''' CREATE table IF NOT EXISTS _cdb_models( name TEXT, - model BLOB, + model TEXT, features TEXT[], accuracy NUMERIC, table_name TEXT, + census_table_name TEXT, + method TEXT )''') def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): """ save a model to the model table for later use """ + create_model_table() plpy.execute(''' - DELETE FROM _cdb_models WHERE model_name = {model_name} + DELETE FROM _cdb_models WHERE name = '{model_name}' '''.format(**locals())) - + model_pickle = pickle.dumps(model) plpy.execute(""" - INSERT INTO _cdb_models ({model_name},{model_pickle},{accuracy}) - """) - -def + INSERT INTO _cdb_models ('{model_name}','{model_pickle}',{accuracy}, '{table_name}', '{census_table}', '{method}') + """.format(**locals())) diff --git a/python/crankshaft/setup.py b/python/crankshaft/setup.py index c0f8c50..07ff9e9 100644 --- a/python/crankshaft/setup.py +++ b/python/crankshaft/setup.py @@ -40,9 +40,8 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.11.0','numpy==1.6.1','scipy==0.17.0'], + install_requires=['pysal==1.11.0','numpy==1.10.1','scipy==0.17.0','pandas','sklearn'], - requires=['pysal', 'numpy'], test_suite='test' ) From 803781e08dc14b88ea3148654ce695a61c3e8f10 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 11 Mar 2016 14:09:53 -0500 Subject: [PATCH 005/183] one stop segmentation shop --- pg/sql/0.0.1/05_segmentation.sql | 16 ++++ .../crankshaft/segmentation/segmentation.py | 81 ++++++++++++++----- 2 files changed, 77 insertions(+), 20 deletions(-) diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql index cce29b6..a2bceed 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -25,3 +25,19 @@ AS $$ # TODO: use named parameters or a dictionary return create_segment('table') $$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION + cdb_create_and_predict_segment ( + segment_name TEXT, + table_name TEXT, + column_name TEXT, + geoid_column TEXT DEFAULT 'geoid', + census_table TEXT DEFAULT 'block_groups' + ) +RETURNS TABLE (the_geom geometry, geoid text, prediction Numeric ) +AS $$ + from crankshaft import segmentation + # TODO: use named parameters or a dictionary + return segmentation.create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') +$$ LANGUAGE plpythonu; diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index 201e7e4..eceb069 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -5,11 +5,15 @@ Segmentation creation and prediction import sklearn import numpy as np import pandas as pd -import pickle +import cPickle import plpy +import sys from sklearn.ensemble import ExtraTreesRegressor from sklearn import metrics +from sklearn.externals import joblib from sklearn.cross_validation import train_test_split +import StringIO +import gzip # High level interface --------------------------------------- @@ -22,10 +26,24 @@ def create_segment(segment_name,table_name,column_name,geoid_column,census_table features = data[data.columns.difference([column_name, 'geoid','the_geom'])] target, mean, std = normalize(data[column_name]) model, accuracy = train_model(target,features, test_split=0.2) - # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) # predict_segment return accuracy +def create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,method): + """ + generate a segment with machine learning + Stuart Lynn + """ + data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) + features = data[data.columns.difference([column_name, 'geoid','the_geom'])] + target, mean, std = normalize(data[column_name]) + model, accuracy, used_features = train_model(target,features, test_split=0.2) + # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) + result = predict_segment(model,used_features,geoid_column,census_table) + return result + + def normalize(target): mean = np.mean(target) std = np.std(target) @@ -37,20 +55,17 @@ def denormalize(target, mean ,std): def train_model(target,features,test_split): plpy.notice('training the model') - plpy.notice('dataframe shape '+ str(np.shape(features))) - plpy.notice('dataframe columns '+ str(features.dtypes)) + plpy.notice('before ', str(np.shape(features))) features = features.dropna(axis =1, how='all').fillna(0) + plpy.notice('after ', str(np.shape(features))) target = target.fillna(0) features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) - plpy.notice('training the model test train split') - model = ExtraTreesRegressor(n_estimators = 40, max_features=len(features.columns)) - plpy.notice('training the model created tree') - plpy.notice('features '+str(np.shape(features_train))+" "+str(np.shape(features_test)) ) - + model = ExtraTreesRegressor(n_estimators = 100, max_features=len(features.columns)) + plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) - plpy.notice('training the model fitting model') + plpy.notice('training the model: fitting one') accuracy = calculate_model_accuracy(model,features,target) - return model, accuracy + return model, accuracy, features.columns def calculate_model_accuracy(model,features,target): prediction = model.predict(features) @@ -82,13 +97,25 @@ def predict_segment(model,features,geoid_column,census_table): predict a segment with machine learning Stuart Lynn """ - data = fetch_model(segment_name) - model = data['model'] - features = ",".join(data['features']) - targets = plpy.execute('select {features} from {census_table}') - geo_ids = plpy.execute('select geoid from {census_table}') - result = model.predict(targets) - return zip(geo_ids,prediction) + # data = fetch_model(segment_name) + # model = data['model'] + # features = ",".join(features) + + joined_features = ','.join(['\"'+a+'\"' for a in features]) + targets = pd.DataFrame(query_to_dictionary(plpy.execute('select {joined_features} from {census_table}'.format(**locals())))) + plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) + plpy.notice(joined_features) + targets = targets.dropna(axis =1, how='all').fillna(0) + plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) + geo_ids = plpy.execute('select geoid from {census_table}'.format(**locals())) + geoms = plpy.execute('select the_geom from {census_table}'.format(**locals())) + + plpy.notice('predicting: predicting data') + + prediction = model.predict(targets) + plpy.notice('predicting: predicted') + + return zip( [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction) def fetch_model(model_name): @@ -127,7 +154,21 @@ def save_model(model_name,model,accuracy,table_name, column_name,census_table,ge plpy.execute(''' DELETE FROM _cdb_models WHERE name = '{model_name}' '''.format(**locals())) - model_pickle = pickle.dumps(model) + + # stringio = StringIO.StringIO() + # gzip_file = gzip.GzipFile(fileobj=stringio, mode='w') + # gzip_file.write() + # gzip_file.close() + + model_pickle = cPickle.dumps(model) #stringio.getvalue() + + + # stringio.close() + + plpy.notice(type(model_pickle)) + plpy.notice(len(model_pickle)) + plpy.notice(sys.getsizeof(model_pickle)) + model_pickle =plpy.quote_literal(model_pickle) plpy.execute(""" - INSERT INTO _cdb_models ('{model_name}','{model_pickle}',{accuracy}, '{table_name}', '{census_table}', '{method}') + INSERT INTO _cdb_models VALUES ('{model_name}',$${model_pickle}$$, Array['test1', 'test2'],{accuracy}, '{table_name}', '{census_table}', '{method}') """.format(**locals())) From f134a54c247527628e3b0ba1eed4a184397b70ab Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 15 Mar 2016 12:44:33 -0400 Subject: [PATCH 006/183] adding one shot train and predict function --- pg/sql/0.0.1/05_segmentation.sql | 7 ++++--- .../crankshaft/segmentation/segmentation.py | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql index a2bceed..1bd97f7 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -2,7 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_create_segment ( segment_name TEXT, table_name TEXT, - column_name TEXT, + column_name TEXT, geoid_column TEXT DEFAULT 'geoid', census_table TEXT DEFAULT 'block_groups' ) @@ -31,7 +31,8 @@ CREATE OR REPLACE FUNCTION cdb_create_and_predict_segment ( segment_name TEXT, table_name TEXT, - column_name TEXT, + column_name TEXT, + target_table TEXT, geoid_column TEXT DEFAULT 'geoid', census_table TEXT DEFAULT 'block_groups' ) @@ -39,5 +40,5 @@ RETURNS TABLE (the_geom geometry, geoid text, prediction Numeric ) AS $$ from crankshaft import segmentation # TODO: use named parameters or a dictionary - return segmentation.create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') + return segmentation.create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,target_table,'random_forest') $$ LANGUAGE plpythonu; diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index eceb069..b7784d9 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -23,24 +23,24 @@ def create_segment(segment_name,table_name,column_name,geoid_column,census_table Stuart Lynn """ data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) - features = data[data.columns.difference([column_name, 'geoid','the_geom'])] + features = data[data.columns.difference([column_name, 'geoid','the_geom', 'the_geom_webmercator'])] target, mean, std = normalize(data[column_name]) model, accuracy = train_model(target,features, test_split=0.2) save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) # predict_segment return accuracy -def create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,method): +def create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,target_table,method): """ generate a segment with machine learning Stuart Lynn """ data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) - features = data[data.columns.difference([column_name, 'geoid','the_geom'])] + features = data[data.columns.difference([column_name, 'the_geom_webmercator', 'geoid','the_geom'])] target, mean, std = normalize(data[column_name]) model, accuracy, used_features = train_model(target,features, test_split=0.2) # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) - result = predict_segment(model,used_features,geoid_column,census_table) + result = predict_segment(model,used_features,geoid_column,target_table) return result @@ -73,8 +73,8 @@ def calculate_model_accuracy(model,features,target): def join_with_census(table_name, column_name, geoid_column, census_table): columns = plpy.execute('select * from {census_table} limit 1 '.format(**locals())) - combined_columns = [ a for a in columns[0].keys() if a not in ['the_geom','cartodb_id','geoid']] - feature_names = ",".join([ " {census_table}.\"{a}\" as \"{a}\" ".format(**locals()) for a in combined_columns]) + combined_columns = [ a for a in columns[0].keys() if a not in ['the_geom','cartodb_id','geoid','the_geom_webmercator']] + feature_names = ",".join([ " {census_table}.\"{a}\"::Numeric as \"{a}\" ".format(**locals()) for a in combined_columns]) plpy.notice('joining with census data') join_data = plpy.execute(''' @@ -101,7 +101,7 @@ def predict_segment(model,features,geoid_column,census_table): # model = data['model'] # features = ",".join(features) - joined_features = ','.join(['\"'+a+'\"' for a in features]) + joined_features = ','.join(['\"'+a+'\"::numeric' for a in features]) targets = pd.DataFrame(query_to_dictionary(plpy.execute('select {joined_features} from {census_table}'.format(**locals())))) plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) plpy.notice(joined_features) From d3e1fca2b3e8761fa4f5930d470e15dc26197ac1 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 16 Mar 2016 15:46:59 -0400 Subject: [PATCH 007/183] changing form of function to use query --- pg/sql/0.0.1/05_segmentation.sql | 5 ++-- .../crankshaft/segmentation/segmentation.py | 27 ++++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql index 1bd97f7..8cbf1c9 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -30,8 +30,7 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_create_and_predict_segment ( segment_name TEXT, - table_name TEXT, - column_name TEXT, + query TEXT, target_table TEXT, geoid_column TEXT DEFAULT 'geoid', census_table TEXT DEFAULT 'block_groups' @@ -40,5 +39,5 @@ RETURNS TABLE (the_geom geometry, geoid text, prediction Numeric ) AS $$ from crankshaft import segmentation # TODO: use named parameters or a dictionary - return segmentation.create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,target_table,'random_forest') + return segmentation.create_and_predict_segment(segment_name,query,geoid_column,census_table,target_table,'random_forest') $$ LANGUAGE plpythonu; diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index b7784d9..248fb99 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -30,14 +30,17 @@ def create_segment(segment_name,table_name,column_name,geoid_column,census_table # predict_segment return accuracy -def create_and_predict_segment(segment_name,table_name,column_name,geoid_column,census_table,target_table,method): +def create_and_predict_segment(segment_name,query,geoid_column,census_table,target_table,method): """ generate a segment with machine learning Stuart Lynn """ - data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) - features = data[data.columns.difference([column_name, 'the_geom_webmercator', 'geoid','the_geom'])] - target, mean, std = normalize(data[column_name]) + data = pd.DataFrame(join_with_census(query,geoid_column, census_table)) + features = data[data.columns.difference(['target', 'the_geom_webmercator', 'geoid','the_geom'])] + target, mean, std = normalize(data['target']) + + normed_target,target_mean, target_std = normalize(target) + model, accuracy, used_features = train_model(target,features, test_split=0.2) # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) result = predict_segment(model,used_features,geoid_column,target_table) @@ -71,17 +74,20 @@ def calculate_model_accuracy(model,features,target): prediction = model.predict(features) return metrics.mean_squared_error(prediction,target)/np.std(target) -def join_with_census(table_name, column_name, geoid_column, census_table): - columns = plpy.execute('select * from {census_table} limit 1 '.format(**locals())) - combined_columns = [ a for a in columns[0].keys() if a not in ['the_geom','cartodb_id','geoid','the_geom_webmercator']] +def join_with_census(query, geoid_column, census_table): + columns = plpy.execute('select * from {census_table} limit 1 '.format(**locals())) + combined_columns = [ a for a in columns[0].keys() if a not in ['target','the_geom','cartodb_id','geoid','the_geom_webmercator']] + plpy.notice(combined_columns) feature_names = ",".join([ " {census_table}.\"{a}\"::Numeric as \"{a}\" ".format(**locals()) for a in combined_columns]) + plpy.notice(feature_names) + plpy.notice('joining with census data') join_data = plpy.execute(''' - SELECT {feature_names}, {table_name}.{column_name} - FROM {table_name} + SELECT {feature_names}, a.target + FROM ({query}) a JOIN {census_table} - ON {table_name}.{geoid_column}::numeric = {census_table}.geoid::numeric + ON a.{geoid_column}::numeric = {census_table}.geoid::numeric '''.format(**locals())) if len(join_data) == 0: @@ -113,6 +119,7 @@ def predict_segment(model,features,geoid_column,census_table): plpy.notice('predicting: predicting data') prediction = model.predict(targets) + de_norm_prediciton = [] plpy.notice('predicting: predicted') return zip( [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction) From fb071215dc7f2445342f17d7910a7f7c5d9001f5 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 17 Mar 2016 19:01:19 -0400 Subject: [PATCH 008/183] changing the function call to use queries --- .../crankshaft/crankshaft/segmentation/segmentation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index 248fb99..b79b47c 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -40,11 +40,11 @@ def create_and_predict_segment(segment_name,query,geoid_column,census_table,targ target, mean, std = normalize(data['target']) normed_target,target_mean, target_std = normalize(target) - + plpy.notice('mean ', target_mean, " std ", target_std) model, accuracy, used_features = train_model(target,features, test_split=0.2) # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) - result = predict_segment(model,used_features,geoid_column,target_table) - return result + geoms, geoids, result = predict_segment(model,used_features,geoid_column,target_table) + return zip(geoms,geoids, [denormalize(t,target_mean, target_std) for t in result] ) def normalize(target): @@ -63,7 +63,7 @@ def train_model(target,features,test_split): plpy.notice('after ', str(np.shape(features))) target = target.fillna(0) features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) - model = ExtraTreesRegressor(n_estimators = 100, max_features=len(features.columns)) + model = ExtraTreesRegressor(n_estimators = 200, max_features=len(features.columns)) plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) plpy.notice('training the model: fitting one') @@ -122,7 +122,7 @@ def predict_segment(model,features,geoid_column,census_table): de_norm_prediciton = [] plpy.notice('predicting: predicted') - return zip( [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction) + return [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction def fetch_model(model_name): From 9b329187460c03383f9002544a1b4357eeb11d07 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 21 Mar 2016 09:50:57 -0400 Subject: [PATCH 009/183] fixing cluster query tests --- src/py/crankshaft/test/test_clustering_moran.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index b48b8d6..6a9107a 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -60,10 +60,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS NOT NULL AND " \ + "FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS NOT NULL AND " \ "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ - "As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT " \ + "As neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT " \ "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ "BY i.\"cartodb_id\" ASC;" @@ -74,10 +74,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE ST_Touches(" \ + "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE ST_Touches(" \ "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ - "neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT NULL " \ + "neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT NULL " \ "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ "i.\"cartodb_id\" ASC;" @@ -88,10 +88,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS " \ + "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS " \ "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ - "OFFSET 1 ) ) As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE " \ + "OFFSET 1 ) ) As neighbors FROM (SELECT * FROM a_list) As i WHERE " \ "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" @@ -106,6 +106,7 @@ class MoranTest(unittest.TestCase): def test_get_weight(self): """Test get_weight.""" + ## need to add tests self.assertEqual(True, True) From 1e16b7839b40ddc88b8f793d17a8201d8c677f36 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:35:39 -0400 Subject: [PATCH 010/183] update methods to acomm markov --- .../crankshaft/crankshaft/clustering/moran.py | 91 +++++++++++-------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 9dd976e..386ab92 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -47,7 +47,7 @@ def moran_local(subquery, attr, significance, num_ngbrs, permutations, geom_colu lisa = ps.Moran_Local(y, w) # find units of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + lisa_sig = quad_position(lisa.q) plpy.notice('** Finished calculations') @@ -95,7 +95,7 @@ def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, w, permutations=permutations) # find units of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + lisa_sig = quad_position(lisa.q) plpy.notice('** Finished calculations') @@ -136,7 +136,7 @@ def moran_local_bv(t, attr1, attr2, significance, num_ngbrs, permutations, geom_ plpy.notice("len of Is: %d" % len(lisa.Is)) # find clustering of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + lisa_sig = quad_position(lisa.q) plpy.notice('** Finished calculations') @@ -169,33 +169,62 @@ def query_attr_select(params): :param params: dict of information used in query (column names, table name, etc.) """ - - attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')] - - template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " - + template = "i.\"%(col)s\"::numeric As attr%(alias_num)s, " attr_string = "" - for idx, val in enumerate(sorted(attrs)): - attr_string += template % {"col": val, "alias_num": idx + 1} + if 'time_cols' in params: + ## if markov analysis + attrs = params['time_cols'] + + for idx, val in enumerate(attrs): + attr_string += template % {"col": val, "alias_num": idx + 1} + else: + ## if moran's analysis + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')] + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": params[val], "alias_num": idx + 1} return attr_string def query_attr_where(params): """ Create portion of WHERE clauses for weeding out NULL-valued geometries + Input: dict of params: + {'subquery': ..., + 'numerator': 'data1', + 'denominator': 'data2', + '': ...} + Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' + + Input: + {'subquery': ..., + 'time_cols': ['time1', 'time2', 'time3'], + 'etc': ...} + Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT NULL AND idx_replace."time3" IS NOT NULL' """ - attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')]) - attr_string = [] + template = "idx_replace.\"%s\" IS NOT NULL" - for attr in attrs: - attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + if 'time_cols' in params: + ## markov where clauses + attrs = params['time_cols'] + # add values to template + for attr in attrs: + attr_string.append(template % attr) + else: + ## moran where clauses - if len(attrs) == 2: - attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + # get keys + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]) + # add values to template + for attr in attrs: + attr_string.append(template % params[attr]) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]]) out = " AND ".join(attr_string) @@ -217,15 +246,16 @@ def knn(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"({subquery})\" As j " \ + "FROM ({subquery}) As j " \ "WHERE %(attr_where_j)s " \ "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ "LIMIT {num_ngbrs} OFFSET 1 ) " \ ") As neighbors " \ - "FROM \"({subquery})\" As i " \ + "FROM ({subquery}) As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements + print query return query.format(**params) @@ -245,11 +275,11 @@ def queen(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"({subquery})\" As j " \ + "FROM ({subquery}) As j " \ "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ "%(attr_where_j)s)" \ ") As neighbors " \ - "FROM \"({subquery})\" As i " \ + "FROM ({subquery}) As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements @@ -302,20 +332,3 @@ def quad_position(quads): lisa_sig = np.array([map_quads(q) for q in quads]) return lisa_sig - -def lisa_sig_vals(pvals, quads, threshold): - """ - Produce Moran's I classification based of n - """ - - sig = (pvals <= threshold) - - lisa_sig = np.empty(len(sig), np.chararray) - - for idx, val in enumerate(sig): - if val: - lisa_sig[idx] = map_quads(quads[idx]) - else: - lisa_sig[idx] = 'Not significant' - - return lisa_sig From 58cf210e966b27fa13ad46af71c42393ce4c5e23 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:37:23 -0400 Subject: [PATCH 011/183] fixes test to acomm markov --- src/py/crankshaft/test/test_clustering_moran.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 6a9107a..4c06594 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -41,17 +41,17 @@ class MoranTest(unittest.TestCase): def test_query_attr_select(self): """Test query_attr_select.""" - ans = "i.\"{attr1}\"::numeric As attr1, " \ - "i.\"{attr2}\"::numeric As attr2, " + ans = "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " self.assertEqual(cc.query_attr_select(self.params), ans) def test_query_attr_where(self): """Test query_attr_where.""" - ans = "idx_replace.\"{attr1}\" IS NOT NULL AND "\ - "idx_replace.\"{attr2}\" IS NOT NULL AND "\ - "idx_replace.\"{attr2}\" <> 0" + ans = "idx_replace.\"andy\" IS NOT NULL AND "\ + "idx_replace.\"jay_z\" IS NOT NULL AND "\ + "idx_replace.\"jay_z\" <> 0" self.assertEqual(cc.query_attr_where(self.params), ans) From b8894169472af5a1d56e74cfd8ea239964c21b7a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:38:44 -0400 Subject: [PATCH 012/183] adding markov functionality --- src/pg/sql/11_markov.sql | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/pg/sql/11_markov.sql diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql new file mode 100644 index 0000000..63a46ac --- /dev/null +++ b/src/pg/sql/11_markov.sql @@ -0,0 +1,81 @@ +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_cols text[], + num_time_per_bin int DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.space_time_predictions import spatial_markov + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + +-- input table format: identical to above but in a predictable format +-- Sample function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- 'date_1') + + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_col_min text, + time_col_max text, + date_format text, -- '_YYYY_MM_DD' + num_time_per_bin INT DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + +-- input table format: +-- id | geom | date | measurement +-- 1 | Pt1 | 12/3 | 13.2 +-- 2 | Pt2 | 11/5 | 11.3 +-- 3 | Pt1 | 11/13 | 12.9 +-- 4 | Pt3 | 12/19 | 10.1 +-- ... + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_col text, + num_time_per_bin INT DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; From c488900c8c1144ca0e6cc73b9d477b951647375a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:39:51 -0400 Subject: [PATCH 013/183] adding markov python code --- .../space_time_dynamics/__init__.py | 1 + .../crankshaft/space_time_dynamics/markov.py | 138 ++++++++++++++++++ .../test/test_space_time_dynamics.py | 126 ++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py create mode 100644 src/py/crankshaft/crankshaft/space_time_dynamics/markov.py create mode 100644 src/py/crankshaft/test/test_space_time_dynamics.py diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py new file mode 100644 index 0000000..f45ffdf --- /dev/null +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -0,0 +1 @@ +from markov import * \ No newline at end of file diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py new file mode 100644 index 0000000..911d558 --- /dev/null +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -0,0 +1,138 @@ +""" +Spatial dynamics measurements using Spatial Markov +""" + + +import numpy as np +import pysal as ps +import plpy +from crankshaft.clustering import get_query + +def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Predict the trends of a unit based on: + 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) + 2. average class of its neighbors + + Inputs: + + @param subquery string: e.g., SELECT * FROM table_name + @param time_cols list (string): list of strings of column names + @param num_time_per_bin int: number of bins to divide # of time columns into + @param permutations int: number of permutations for test stats + @param geom_col string: name of column which contains the geometries + @param id_col string: name of column which has the ids of the table + @param w_type string: weight type ('knn' or 'queen') + @param num_ngbrs int: number of neighbors (if knn type) + + Outputs: + @param trend_up float: probablity that a geom will move to a higher class + @param trend_down float: probablity that a geom will move to a lower class + @param trend float: (trend_up - trend_down) / trend_static + @param volatility float: a measure of the volatility based on probability stddev(prob array) + @param + """ + + qvals = {"id_col": id_col, + "time_cols": time_cols, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = get_query(w_type, qvals) + + try: + query_result = plpy.execute(query) + except: + zip([None],[None],[None]) + + ## build weight + weights = get_weight(query_result, w_type) + + ## prep time data + t_data = get_time_data(query_result, time_cols) + ## rebin time data + if num_time_per_bin > 1: + ## rebin + t_data = rebin_data(t_data, num_time_per_bin) + + sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) + + ## get lags + lags = ps.lag_spatial(weights, t_data) + + ## get lag classes + lag_classes = ps.Quantiles(lags.flatten(), k=7).yb + + ## look up probablity distribution for each unit according to class and lag class + prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) + + ## find the ups and down and overall distribution of each cell + trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + + ## output the results + + return zip(trend, trend_up, trend_down, volatility, weights.id_order) + +def get_time_data(markov_data, time_cols): + """ + Extract the time columns and bin appropriately + """ + return np.array([[x[t_col] for x in query_result] for t_col in time_cols], dtype=float) + +def rebin_data(time_data, num_time_per_bin): + """ + convert an n x l matrix into an (n/m) x l matrix where the values are reduced (averaged) for the intervening states: + 1 2 3 4 1.5 3.5 + 5 6 7 8 -> 5.5 7.5 + 9 8 7 6 8.5 6.5 + 5 4 3 2 4.5 2.5 + + if m = 2 + + This process effectively resamples the data at a longer time span n units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining two columns are binned together as the last time period, while the first three are binned together. + + Input: + @param time_data n x l ndarray: measurements of an attribute at different time intervals + @param num_time_per_bin int: number of columns to average into a new column + Output: + ceil(n / m) x l ndarray of resampled time series + """ + + if time_data.shape[1] % num_time_per_bin == 0: + ## if fit is perfect, then use it + n_max = time_data.shape[1] / num_time_per_bin + else: + ## fit remainders into an additional column + n_max = time_data.shape[1] / num_time_per_bin + 1 + + return np.array([ + time_data[:, + num_time_per_bin*i:num_time_per_bin*(i+1)].mean(axis=1) + for i in range(n_max)]).T +def get_prob_dist(transition_matrix, lag_indices, unit_indices): + """ + given an array of transition matrices, look up the probability associated with the arrangements passed + + Input: + @param transition_matrix ndarray[k,k,k]: + @param lag_indices ndarray: + @param unit_indices ndarray: + + Output: + Array of probability distributions + """ + + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) + +def get_prob_stats(prob_dist, unit_indices): +# trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + + trend_up = np.array([prob_dist[:, i:].sum() for i in unit_indices]) + trend_down = np.array([prob_dist[:, :i].sum() for i in unit_indices]) + trend = trend_up - trend_down + volatility = prob_dist.std(axis=1) + + + return trend_up, trend_down, trend, volatility diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py new file mode 100644 index 0000000..dd7b8b0 --- /dev/null +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -0,0 +1,126 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.space_time_dynamics as std +import crankshaft.clustering as cc +from crankshaft import random_seeds +import json + +class SpaceTimeTests(unittest.TestCase): + """Testing class for Markov Functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T + + self.transition_matrix = p = np.array([ + [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], + [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], + [ 0. , 0.14 , 0.74 , 0.12 , 0. ], + [ 0. , 0.03571429, 0.32142857, 0.57142857, 0.07142857], + [ 0. , 0. , 0. , 0.16666667, 0.83333333]], + [[ 0.79831933, 0.16806723, 0.03361345, 0. , 0. ], + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0.00537634, 0.06989247, 0.8655914 , 0.05913978, 0. ], + [ 0. , 0. , 0.06372549, 0.90196078, 0.03431373], + [ 0. , 0. , 0. , 0.19444444, 0.80555556]], + [[ 0.84693878, 0.15306122, 0. , 0. , 0. ], + [ 0.08133971, 0.78947368, 0.1291866 , 0. , 0. ], + [ 0.00518135, 0.0984456 , 0.79274611, 0.0984456 , 0.00518135], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0. , 0. , 0. , 0.10204082, 0.89795918]], + [[ 0.8852459 , 0.09836066, 0. , 0.01639344, 0. ], + [ 0.03875969, 0.81395349, 0.13953488, 0. , 0.00775194], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0.02339181, 0.12865497, 0.75438596, 0.09356725], + [ 0. , 0. , 0. , 0.09661836, 0.90338164]], + [[ 0.33333333, 0.66666667, 0. , 0. , 0. ], + [ 0.0483871 , 0.77419355, 0.16129032, 0.01612903, 0. ], + [ 0.01149425, 0.16091954, 0.74712644, 0.08045977, 0. ], + [ 0. , 0.01036269, 0.06217617, 0.89637306, 0.03108808], + [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] + ) + + # def test_spatial_markov(self): + # """Test Spatial Markov.""" + # + # ans = "SELECT i.\"cartodb_id\" As id, " \ + # "i.\"dec_2013\"::numeric As attr1, " \ + # "i.\"jan_2014\"::numeric As attr2, " \ + # "i.\"feb_2014\"::numeric As attr3, " \ + # "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + # "FROM (SELECT * FROM a_list) As j " \ + # "WHERE j.\"dec_2013\" IS NOT NULL AND " \ + # "j.\"jan_2014\" IS NOT NULL AND " \ + # "j.\"feb_2014\" IS NOT NULL " \ + # "ORDER BY " \ + # "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + # "LIMIT 321 OFFSET 1 ) ) " \ + # "As neighbors " \ + # "FROM (SELECT * FROM a_list) As i " \ + # "WHERE i.\"dec_2013\" IS NOT NULL AND " \ + # "i.\"jan_2014\" IS NOT NULL AND " \ + # "i.\"feb_2014\" IS NOT NULL " \ + # "ORDER BY i.\"cartodb_id\" ASC;" + # + # subquery = self.params['subquery'] + # time_cols = self.params['time_cols'] + # num_time_per_bin = 1 + # permutations = 99 + # geom_col = self.params['geom_col'] + # id_col = self.params['id_col'] + # w_type = 'knn' + # num_ngbrs = self.params['num_ngbrs'] + # + # self.assertEqual(std.spatial_markov(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs), ans) + + def test_rebin_data(self): + """Test rebin_data""" + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## = 0.5, 2.5, 4.5, 6.5, 8.5 + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + for i in range(0, 10, 2)]).T + + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) + + ## sample in triple the time (uneven since 10 % 3 = 1): + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## = 1, 4, 7, 9 + ans_odd = np.array([i * np.ones(10, dtype=float) + for i in (1, 4, 7, 9)]).T + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) + def test_get_prob_dist(self): + """Test get_prob_dist""" + lag_indices = np.array([1, 2, 3, 4]) + unit_indices = np.array([1, 3, 2, 4]) + answer = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) + + self.assertTrue(np.array_equal(result, answer)) + + + From f3673d6f89631fbbae1b046996795184f374215e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:41:15 -0400 Subject: [PATCH 014/183] updating test for moran output --- src/pg/test/expected/02_moran_test.out | 121 +++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 66ccaaa..8644e6b 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -126,13 +126,65 @@ SELECT ppoints.code, m.quads ORDER BY ppoints.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."value"::numeric As attr1, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints)" As j WHERE j."value" IS NOT NULL ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints)" As i WHERE i."value" IS NOT NULL ORDER BY i."cartodb_id" ASC;" +NOTICE: ** Query returned with 52 rows CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Exiting function +NOTICE: ** Finished calculations CONTEXT: PL/Python function "cdb_moran_local" code | quads ------+------- -(0 rows) + 01 | HH + 02 | HL + 03 | LL + 04 | LL + 05 | LH + 06 | LL + 07 | HH + 08 | HH + 09 | HH + 10 | LL + 11 | LL + 12 | LL + 13 | HL + 14 | LL + 15 | LL + 16 | HH + 17 | HH + 18 | LL + 19 | HH + 20 | HH + 21 | LL + 22 | HH + 23 | LL + 24 | LL + 25 | HH + 26 | HH + 27 | LL + 28 | HH + 29 | LL + 30 | LL + 31 | HH + 32 | LL + 33 | HL + 34 | LH + 35 | LL + 36 | LL + 37 | HL + 38 | HL + 39 | HH + 40 | HH + 41 | HL + 42 | LH + 43 | LH + 44 | LL + 45 | LH + 46 | LL + 47 | LL + 48 | HH + 49 | LH + 50 | HH + 51 | LL + 52 | LL +(52 rows) SELECT cdb_crankshaft._cdb_random_seeds(1234); _cdb_random_seeds @@ -147,12 +199,61 @@ SELECT ppoints2.code, m.quads ORDER BY ppoints2.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."denominator"::numeric As attr1, i."numerator"::numeric As attr2, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints2)" As j WHERE j."denominator" IS NOT NULL AND j."numerator" IS NOT NULL AND j."numerator" <> 0 ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints2)" As i WHERE i."denominator" IS NOT NULL AND i."numerator" IS NOT NULL AND i."numerator" <> 0 ORDER BY i."cartodb_id" ASC;" +NOTICE: ** Query returned with 51 rows CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Error: +NOTICE: ** Finished calculations CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Exiting function -CONTEXT: PL/Python function "cdb_moran_local_rate" -ERROR: length of returned sequence did not match number of columns in row -CONTEXT: while creating return value -PL/Python function "cdb_moran_local_rate" + code | quads +------+------- + 01 | LL + 02 | LH + 03 | HH + 04 | HH + 05 | LL + 06 | HH + 07 | LL + 08 | LL + 09 | LL + 10 | HH + 11 | HH + 12 | HL + 13 | LL + 14 | HH + 15 | LL + 16 | LL + 17 | LL + 18 | LH + 19 | LL + 20 | LL + 21 | HH + 22 | LL + 23 | HL + 24 | LL + 25 | LL + 26 | LL + 27 | LL + 28 | LL + 29 | LH + 30 | HH + 31 | LL + 32 | LL + 33 | LL + 34 | LL + 35 | LH + 36 | HL + 37 | LH + 38 | LH + 39 | LL + 40 | LL + 41 | LH + 42 | HL + 43 | LL + 44 | HL + 45 | LL + 46 | HL + 47 | LL + 48 | LL + 49 | HL + 50 | LL + 51 | HH +(51 rows) From cfb40ddecd20556781687745f45037f6e3b9632f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 13:01:56 -0400 Subject: [PATCH 015/183] fixes test for moran --- src/pg/test/expected/02_moran_test.out | 1 + src/py/crankshaft/test/fixtures/moran.json | 68 +++++++++++----------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 8644e6b..92cb218 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -257,3 +257,4 @@ CONTEXT: PL/Python function "cdb_moran_local_rate" 50 | LL 51 | HH (51 rows) + diff --git a/src/py/crankshaft/test/fixtures/moran.json b/src/py/crankshaft/test/fixtures/moran.json index 0530c18..9bd90d7 100644 --- a/src/py/crankshaft/test/fixtures/moran.json +++ b/src/py/crankshaft/test/fixtures/moran.json @@ -1,52 +1,52 @@ [[0.9319096128346788, "HH"], [-1.135787401862846, "HL"], -[0.11732030672508517, "Not significant"], -[0.6152779669180425, "Not significant"], -[-0.14657336660125297, "Not significant"], -[0.6967858120189607, "Not significant"], -[0.07949310115714454, "Not significant"], -[0.4703198759258987, "Not significant"], -[0.4421125200498064, "Not significant"], -[0.5724288737143592, "Not significant"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], [0.8970743435692062, "LL"], -[0.18327334401918674, "Not significant"], -[-0.01466729201304962, "Not significant"], -[0.3481559372544409, "Not significant"], -[0.06547094736902978, "Not significant"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], [0.15482141569329988, "HH"], -[0.4373841193538136, "Not significant"], -[0.15971286468915544, "Not significant"], -[1.0543588860308968, "Not significant"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], [1.7372866900020818, "HH"], [1.091998586053999, "LL"], -[0.1171572584252222, "Not significant"], -[0.08438455015300014, "Not significant"], -[0.06547094736902978, "Not significant"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], [0.15482141569329985, "HH"], [1.1627044812890683, "HH"], -[0.06547094736902978, "Not significant"], -[0.795275137550483, "Not significant"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], [0.18562939195219, "LL"], -[0.3010757406693439, "Not significant"], +[0.3010757406693439, "LL"], [2.8205795942839376, "HH"], -[0.11259190602909264, "Not significant"], -[-0.07116352791516614, "Not significant"], -[-0.09945240794119009, "Not significant"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], [0.18562939195219, "LL"], -[0.1832733440191868, "Not significant"], -[-0.39054253768447705, "Not significant"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], [-0.1672071289487642, "HL"], -[0.3337669247916343, "Not significant"], -[0.2584386102554792, "Not significant"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], [-0.19733845476322634, "HL"], [-0.9379282899805409, "LH"], -[-0.028770969951095866, "Not significant"], -[0.051367269430983485, "Not significant"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], [-0.2172548045913472, "LH"], -[0.05136726943098351, "Not significant"], -[0.04191046803899837, "Not significant"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], [0.7482357030403517, "HH"], -[-0.014585767863118111, "Not significant"], -[0.5410013139159929, "Not significant"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], [1.0223932668429925, "LL"], [1.4179402898927476, "LL"]] From 42e760b5d104b57f42307e7903af3f63cd4c9fce Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 14:50:52 -0400 Subject: [PATCH 016/183] adding passing tests --- .../crankshaft/space_time_dynamics/markov.py | 31 ++++++++--- .../test/test_space_time_dynamics.py | 55 +++++++++++++------ 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 911d558..db2be6f 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -54,7 +54,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge ## rebin time data if num_time_per_bin > 1: ## rebin - t_data = rebin_data(t_data, num_time_per_bin) + t_data = rebin_data(t_data, int(num_time_per_bin)) sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) @@ -68,7 +68,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) ## find the ups and down and overall distribution of each cell - trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist) ## output the results @@ -127,12 +127,29 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices): return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) def get_prob_stats(prob_dist, unit_indices): -# trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + """ + get the statistics of the probability distributions - trend_up = np.array([prob_dist[:, i:].sum() for i in unit_indices]) - trend_down = np.array([prob_dist[:, :i].sum() for i in unit_indices]) - trend = trend_up - trend_down + Outputs: + @param trend_up ndarray(float): sum of probabilities for upward + movement (relative to the unit index of that prob) + @param trend_down ndarray(float): sum of probabilities for downard + movement (relative to the unit index of that prob) + @param trend ndarray(float): difference of upward and downward + movements + """ + + num_elements = len(prob_dist) + trend_up = np.empty(num_elements) + trend_down = np.empty(num_elements) + trend = np.empty(num_elements) + + for i in range(num_elements): + trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() + trend_down[i] = prob_dist[i, :unit_indices[i]].sum() + trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + + ## calculate volatility of distribution volatility = prob_dist.std(axis=1) - return trend_up, trend_down, trend, volatility diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py index dd7b8b0..c35aea5 100644 --- a/src/py/crankshaft/test/test_space_time_dynamics.py +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -28,9 +28,9 @@ class SpaceTimeTests(unittest.TestCase): "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) self.moran_data = json.loads(open(fixture_file('moran.json')).read()) - + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T - + self.transition_matrix = p = np.array([ [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], @@ -61,7 +61,7 @@ class SpaceTimeTests(unittest.TestCase): # def test_spatial_markov(self): # """Test Spatial Markov.""" - # + # # ans = "SELECT i.\"cartodb_id\" As id, " \ # "i.\"dec_2013\"::numeric As attr1, " \ # "i.\"jan_2014\"::numeric As attr2, " \ @@ -80,7 +80,7 @@ class SpaceTimeTests(unittest.TestCase): # "i.\"jan_2014\" IS NOT NULL AND " \ # "i.\"feb_2014\" IS NOT NULL " \ # "ORDER BY i.\"cartodb_id\" ASC;" - # + # # subquery = self.params['subquery'] # time_cols = self.params['time_cols'] # num_time_per_bin = 1 @@ -89,23 +89,23 @@ class SpaceTimeTests(unittest.TestCase): # id_col = self.params['id_col'] # w_type = 'knn' # num_ngbrs = self.params['num_ngbrs'] - # + # # self.assertEqual(std.spatial_markov(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs), ans) - + def test_rebin_data(self): """Test rebin_data""" - ## sample in double the time (even case since 10 % 2 = 0): - ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 ## = 0.5, 2.5, 4.5, 6.5, 8.5 - ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) for i in range(0, 10, 2)]).T - + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) ## sample in triple the time (uneven since 10 % 3 = 1): - ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 ## = 1, 4, 7, 9 - ans_odd = np.array([i * np.ones(10, dtype=float) + ans_odd = np.array([i * np.ones(10, dtype=float) for i in (1, 4, 7, 9)]).T self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) def test_get_prob_dist(self): @@ -119,8 +119,31 @@ class SpaceTimeTests(unittest.TestCase): [ 0. , 0. , 0. , 0.02352941, 0.97647059] ]) result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) - + self.assertTrue(np.array_equal(result, answer)) - - - + + def test_get_prob_stats(self): + """Test get_prob_stats""" + + probs = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + unit_indices = np.array([1, 3, 2, 4]) + answer_up = np.array([0.04245283, 0.03529412, 0.12376238, 0.]) + answer_down = np.array([0.0754717, 0.09411765, 0.0990099, 0.02352941]) + answer_trend = np.array([-0.03301887 / 0.88207547, -0.05882353 / 0.87058824, 0.02475248 / 0.77722772, -0.02352941 / 0.97647059]) + answer_volatility = np.array([ 0.34221495, 0.33705421, 0.29226542, 0.38834223]) + + result = std.get_prob_stats(probs, unit_indices) + result_up = result[0] + result_down = result[1] + result_trend = result[2] + result_volatility = result[3] + + self.assertTrue(np.allclose(result_up, answer_up)) + self.assertTrue(np.allclose(result_down, answer_down)) + self.assertTrue(np.allclose(result_trend, answer_trend)) + self.assertTrue(np.allclose(result_volatility, answer_volatility)) From c5a58f97ecc8af419c6ba40f9e7166f2ada46416 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 15:33:54 -0400 Subject: [PATCH 017/183] add types to arrays --- src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index db2be6f..e809632 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -140,9 +140,9 @@ def get_prob_stats(prob_dist, unit_indices): """ num_elements = len(prob_dist) - trend_up = np.empty(num_elements) - trend_down = np.empty(num_elements) - trend = np.empty(num_elements) + trend_up = np.empty(num_elements, dtype=float) + trend_down = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) for i in range(num_elements): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() From 68e5e0892ccae0a7614fdee8601ef28544f93837 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 20:46:19 -0400 Subject: [PATCH 018/183] update signature used in plpython function --- src/pg/sql/11_markov.sql | 96 ++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 63a46ac..fa7c838 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -6,7 +6,7 @@ -- 2 | Pt2 | 11.0 | 13.2 | 12.5 -- ... -- Sample Function call: --- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', -- Array['date_1', 'date_2', 'date_3']) @@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION time_cols text[], num_time_per_bin int DEFAULT 1, permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', + geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs int DEFAULT 5) @@ -25,7 +25,7 @@ AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.space_time_predictions import spatial_markov # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) + return def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format @@ -34,48 +34,48 @@ $$ LANGUAGE plpythonu; -- 'date_1') -CREATE OR REPLACE FUNCTION - cdb_spatial_markov ( - subquery TEXT, - time_col_min text, - time_col_max text, - date_format text, -- '_YYYY_MM_DD' - num_time_per_bin INT DEFAULT 1, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) -$$ LANGUAGE plpythonu; - --- input table format: --- id | geom | date | measurement --- 1 | Pt1 | 12/3 | 13.2 --- 2 | Pt2 | 11/5 | 11.3 --- 3 | Pt1 | 11/13 | 12.9 --- 4 | Pt3 | 12/19 | 10.1 --- ... - -CREATE OR REPLACE FUNCTION - cdb_spatial_markov ( - subquery TEXT, - time_col text, - num_time_per_bin INT DEFAULT 1, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) -$$ LANGUAGE plpythonu; +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col_min text, +-- time_col_max text, +-- date_format text, -- '_YYYY_MM_DD' +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- +-- -- input table format: +-- -- id | geom | date | measurement +-- -- 1 | Pt1 | 12/3 | 13.2 +-- -- 2 | Pt2 | 11/5 | 11.3 +-- -- 3 | Pt1 | 11/13 | 12.9 +-- -- 4 | Pt3 | 12/19 | 10.1 +-- -- ... +-- +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col text, +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; From 491577ed62d35346eb65be0e0c91ade1dc21e6bb Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 08:19:11 -0400 Subject: [PATCH 019/183] formated markov sql file a bit --- src/pg/sql/11_markov.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index fa7c838..f63af5d 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -23,9 +23,12 @@ CREATE OR REPLACE FUNCTION RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.space_time_predictions import spatial_markov - # TODO: use named parameters or a dictionary - return def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + + return spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format From 98c2b11935a45ffe21f5aaee5681bf791a133090 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 11:33:47 -0400 Subject: [PATCH 020/183] update output signature --- src/pg/sql/11_markov.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index f63af5d..09cca98 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -20,7 +20,7 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +RETURNS TABLE (trend numeric, trend_up numeric, trend_down numeric, volatility numeric, ids int) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') From fbc30f12245928053968ac0f7c6bfb6253f8e462 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 11:34:28 -0400 Subject: [PATCH 021/183] add working version --- .../crankshaft/space_time_dynamics/markov.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index e809632..60321e3 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -6,7 +6,7 @@ Spatial dynamics measurements using Spatial Markov import numpy as np import pysal as ps import plpy -from crankshaft.clustering import get_query +from crankshaft.clustering import get_query, get_weight def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): """ @@ -44,7 +44,9 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge try: query_result = plpy.execute(query) except: - zip([None],[None],[None]) + plpy.notice('** Query failed: %s' % query) + plpy.error('Query failed: check the input parameters') + return zip([None], [None], [None], [None], [None]) ## build weight weights = get_weight(query_result, w_type) @@ -58,17 +60,17 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) - ## get lags - lags = ps.lag_spatial(weights, t_data) + ## get lags of last time slice + lags = ps.lag_spatial(weights, t_data[:, -1]) ## get lag classes - lag_classes = ps.Quantiles(lags.flatten(), k=7).yb + lag_classes = ps.Quantiles(lags, k=7).yb ## look up probablity distribution for each unit according to class and lag class - prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) + prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) ## find the ups and down and overall distribution of each cell - trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist) + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1]) ## output the results @@ -78,7 +80,9 @@ def get_time_data(markov_data, time_cols): """ Extract the time columns and bin appropriately """ - return np.array([[x[t_col] for x in query_result] for t_col in time_cols], dtype=float) + num_attrs = len(time_cols) + return np.array([[x['attr' + str(i)] for x in markov_data] + for i in range(1, num_attrs+1)], dtype=float).T def rebin_data(time_data, num_time_per_bin): """ @@ -139,7 +143,7 @@ def get_prob_stats(prob_dist, unit_indices): movements """ - num_elements = len(prob_dist) + num_elements = len(unit_indices) trend_up = np.empty(num_elements, dtype=float) trend_down = np.empty(num_elements, dtype=float) trend = np.empty(num_elements, dtype=float) From d39849472074ae05701f175779a01d703dd934ac Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 11:34:59 -0400 Subject: [PATCH 022/183] add test case for markov use of functions --- .../crankshaft/test/test_clustering_moran.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 4c06594..f987239 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -26,6 +26,11 @@ class MoranTest(unittest.TestCase): "subquery": "SELECT * FROM a_list", "geom_col": "the_geom", "num_ngbrs": 321} + self.params_markov = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) self.moran_data = json.loads(open(fixture_file('moran.json')).read()) @@ -67,8 +72,27 @@ class MoranTest(unittest.TestCase): "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ "BY i.\"cartodb_id\" ASC;" + ans_markov = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE j.\"_2013_dec\" IS NOT NULL AND " \ + "j.\"_2014_jan\" IS NOT NULL AND " \ + "j.\"_2014_feb\" IS NOT NULL " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321 OFFSET 1 ) ) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"_2013_dec\" IS NOT NULL AND " \ + "i.\"_2014_jan\" IS NOT NULL AND " \ + "i.\"_2014_feb\" IS NOT NULL "\ + "ORDER BY i.\"cartodb_id\" ASC;" + self.assertEqual(cc.knn(self.params), ans) + self.assertEqual(cc.knn(self.params_markov), ans_markov) + def test_queen(self): """Test queen neighbors function.""" From d140b4249e80f3eb95c6e97dddeb0e392f5cbfb8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 24 Mar 2016 14:05:17 -0400 Subject: [PATCH 023/183] updating to use iterative query on prediction --- .../crankshaft/segmentation/segmentation.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index b79b47c..130c8e4 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -98,6 +98,15 @@ def join_with_census(query, geoid_column, census_table): def query_to_dictionary(result): return [ dict(zip(r.keys(), r.values())) for r in result ] +def query_in_batches(query,batch_size): + cursor = plpy.cursor(query) + while True: + rows = cursor.fetch(batch_size) + if not rows: + break + else: + yield query_to_dictionary(rows) + def predict_segment(model,features,geoid_column,census_table): """ predict a segment with machine learning @@ -109,20 +118,20 @@ def predict_segment(model,features,geoid_column,census_table): joined_features = ','.join(['\"'+a+'\"::numeric' for a in features]) targets = pd.DataFrame(query_to_dictionary(plpy.execute('select {joined_features} from {census_table}'.format(**locals())))) - plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) - plpy.notice(joined_features) - targets = targets.dropna(axis =1, how='all').fillna(0) - plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) + + predition = [] + for batch in query_in_batches('select {joined_features} from {census_table}'.format(**locals()),2000): + targets = pd.DataFrame(batch) + plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) + plpy.notice(joined_features) + targets = targets.dropna(axis =1, how='all').fillna(0) + plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) + batch_prediction = model.predict(targets) + prediciton.append(batch_prediction.to_maxtrix) + geo_ids = plpy.execute('select geoid from {census_table}'.format(**locals())) - geoms = plpy.execute('select the_geom from {census_table}'.format(**locals())) - plpy.notice('predicting: predicting data') - - prediction = model.predict(targets) - de_norm_prediciton = [] - plpy.notice('predicting: predicted') - - return [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction + return [[a['geoid'] for a in geo_ids],prediction] def fetch_model(model_name): From 2e1b598b4fba78f42a36c8d049e729f67b44f45b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 25 Mar 2016 22:49:29 -0400 Subject: [PATCH 024/183] pylinting changes --- .../crankshaft/space_time_dynamics/markov.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 60321e3..db2425c 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -8,7 +8,8 @@ import pysal as ps import plpy from crankshaft.clustering import get_query, get_weight -def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): +def spatial_markov_trend(subquery, time_cols, num_time_per_bin, + permutations, geom_col, id_col, w_type, num_ngbrs): """ Predict the trends of a unit based on: 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) @@ -33,6 +34,9 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge @param """ + if num_time_per_bin < 1: + plpy.error('Error: number of time bins must be >= 1') + qvals = {"id_col": id_col, "time_cols": time_cols, "geom_col": geom_col, @@ -58,13 +62,14 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge ## rebin t_data = rebin_data(t_data, int(num_time_per_bin)) - sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) - - ## get lags of last time slice - lags = ps.lag_spatial(weights, t_data[:, -1]) + sp_markov_result = ps.Spatial_Markov(t_data, + weights, + k=7, + fixed=False, + permutations=permutations) ## get lag classes - lag_classes = ps.Quantiles(lags, k=7).yb + lag_classes = ps.Quantiles(ps.lag_spatial(weights, t_data[:, -1]), k=7).yb ## look up probablity distribution for each unit according to class and lag class prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) @@ -86,7 +91,8 @@ def get_time_data(markov_data, time_cols): def rebin_data(time_data, num_time_per_bin): """ - convert an n x l matrix into an (n/m) x l matrix where the values are reduced (averaged) for the intervening states: + convert an n x l matrix into an (n/m) x l matrix where the values are + reduced (averaged) for the intervening states: 1 2 3 4 1.5 3.5 5 6 7 8 -> 5.5 7.5 9 8 7 6 8.5 6.5 @@ -94,12 +100,17 @@ def rebin_data(time_data, num_time_per_bin): if m = 2 - This process effectively resamples the data at a longer time span n units longer than the input data. - For cases when there is a remainder (remainder(5/3) = 2), the remaining two columns are binned together as the last time period, while the first three are binned together. + This process effectively resamples the data at a longer time span n + units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining + two columns are binned together as the last time period, while the + first three are binned together. Input: - @param time_data n x l ndarray: measurements of an attribute at different time intervals - @param num_time_per_bin int: number of columns to average into a new column + @param time_data n x l ndarray: measurements of an attribute at + different time intervals + @param num_time_per_bin int: number of columns to average into a new + column Output: ceil(n / m) x l ndarray of resampled time series """ @@ -111,13 +122,12 @@ def rebin_data(time_data, num_time_per_bin): ## fit remainders into an additional column n_max = time_data.shape[1] / num_time_per_bin + 1 - return np.array([ - time_data[:, - num_time_per_bin*i:num_time_per_bin*(i+1)].mean(axis=1) + return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) for i in range(n_max)]).T def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ - given an array of transition matrices, look up the probability associated with the arrangements passed + given an array of transition matrices, look up the probability + associated with the arrangements passed Input: @param transition_matrix ndarray[k,k,k]: @@ -128,7 +138,8 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices): Array of probability distributions """ - return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] + for i in range(len(lag_indices))]) def get_prob_stats(prob_dist, unit_indices): """ @@ -144,9 +155,9 @@ def get_prob_stats(prob_dist, unit_indices): """ num_elements = len(unit_indices) - trend_up = np.empty(num_elements, dtype=float) + trend_up = np.empty(num_elements, dtype=float) trend_down = np.empty(num_elements, dtype=float) - trend = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) for i in range(num_elements): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() From b8330dce076ad91a3dbdce0b7938f61dad9c0da3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 29 Mar 2016 14:53:51 +0200 Subject: [PATCH 025/183] Fix deployment Makefile goal The virtual environment virtual directory wasn't set properly for deployment. Fixes #21 --- Makefile.global | 11 ++++++----- src/pg/Makefile | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.global b/Makefile.global index 77f6c69..504886f 100644 --- a/Makefile.global +++ b/Makefile.global @@ -1,6 +1,7 @@ SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) -EXTENSION = crankshaft -PACKAGE = crankshaft -EXTVERSION = $(shell grep default_version $(SELF_DIR)/src/pg/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -RELEASE_VERSION ?= $(EXTVERSION) -SED = sed +EXTENSION = crankshaft +PACKAGE = crankshaft +EXTVERSION = $(shell grep default_version $(SELF_DIR)/src/pg/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +RELEASE_VERSION ?= $(EXTVERSION) +SED = sed +VIRTUALENV_PATH := $(realpath $(SELF_DIR)/envs) diff --git a/src/pg/Makefile b/src/pg/Makefile index 8a745c4..f4a34a2 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -18,7 +18,6 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../../envs) ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ From a0cb699b1a999f7753317b57f38e0826f2ca6c92 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 16:06:44 -0400 Subject: [PATCH 026/183] updated move to pysal-utils --- src/py/crankshaft/crankshaft/clustering/moran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 2a043c3..ff8ae00 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -174,7 +174,7 @@ def moran_local_rate(subquery, numerator, denominator, lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, permutations=permutations) - # find units of significance + # find quadrants for each geometry quads = quad_position(lisa.q) return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) From 8cccb18eedd6e61e8f951e576be805b0466c7715 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:38:44 -0400 Subject: [PATCH 027/183] adding markov functionality --- src/pg/sql/11_markov.sql | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/pg/sql/11_markov.sql diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql new file mode 100644 index 0000000..63a46ac --- /dev/null +++ b/src/pg/sql/11_markov.sql @@ -0,0 +1,81 @@ +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_cols text[], + num_time_per_bin int DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.space_time_predictions import spatial_markov + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + +-- input table format: identical to above but in a predictable format +-- Sample function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- 'date_1') + + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_col_min text, + time_col_max text, + date_format text, -- '_YYYY_MM_DD' + num_time_per_bin INT DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + +-- input table format: +-- id | geom | date | measurement +-- 1 | Pt1 | 12/3 | 13.2 +-- 2 | Pt2 | 11/5 | 11.3 +-- 3 | Pt1 | 11/13 | 12.9 +-- 4 | Pt3 | 12/19 | 10.1 +-- ... + +CREATE OR REPLACE FUNCTION + cdb_spatial_markov ( + subquery TEXT, + time_col text, + num_time_per_bin INT DEFAULT 1, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs int DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; From 9943d4de58b684ac2b748a667c2b76374262ad23 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 12:39:51 -0400 Subject: [PATCH 028/183] adding markov python code --- .../space_time_dynamics/__init__.py | 1 + .../crankshaft/space_time_dynamics/markov.py | 138 ++++++++++++++++++ .../test/test_space_time_dynamics.py | 126 ++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py create mode 100644 src/py/crankshaft/crankshaft/space_time_dynamics/markov.py create mode 100644 src/py/crankshaft/test/test_space_time_dynamics.py diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py new file mode 100644 index 0000000..f45ffdf --- /dev/null +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -0,0 +1 @@ +from markov import * \ No newline at end of file diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py new file mode 100644 index 0000000..911d558 --- /dev/null +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -0,0 +1,138 @@ +""" +Spatial dynamics measurements using Spatial Markov +""" + + +import numpy as np +import pysal as ps +import plpy +from crankshaft.clustering import get_query + +def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Predict the trends of a unit based on: + 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) + 2. average class of its neighbors + + Inputs: + + @param subquery string: e.g., SELECT * FROM table_name + @param time_cols list (string): list of strings of column names + @param num_time_per_bin int: number of bins to divide # of time columns into + @param permutations int: number of permutations for test stats + @param geom_col string: name of column which contains the geometries + @param id_col string: name of column which has the ids of the table + @param w_type string: weight type ('knn' or 'queen') + @param num_ngbrs int: number of neighbors (if knn type) + + Outputs: + @param trend_up float: probablity that a geom will move to a higher class + @param trend_down float: probablity that a geom will move to a lower class + @param trend float: (trend_up - trend_down) / trend_static + @param volatility float: a measure of the volatility based on probability stddev(prob array) + @param + """ + + qvals = {"id_col": id_col, + "time_cols": time_cols, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = get_query(w_type, qvals) + + try: + query_result = plpy.execute(query) + except: + zip([None],[None],[None]) + + ## build weight + weights = get_weight(query_result, w_type) + + ## prep time data + t_data = get_time_data(query_result, time_cols) + ## rebin time data + if num_time_per_bin > 1: + ## rebin + t_data = rebin_data(t_data, num_time_per_bin) + + sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) + + ## get lags + lags = ps.lag_spatial(weights, t_data) + + ## get lag classes + lag_classes = ps.Quantiles(lags.flatten(), k=7).yb + + ## look up probablity distribution for each unit according to class and lag class + prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) + + ## find the ups and down and overall distribution of each cell + trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + + ## output the results + + return zip(trend, trend_up, trend_down, volatility, weights.id_order) + +def get_time_data(markov_data, time_cols): + """ + Extract the time columns and bin appropriately + """ + return np.array([[x[t_col] for x in query_result] for t_col in time_cols], dtype=float) + +def rebin_data(time_data, num_time_per_bin): + """ + convert an n x l matrix into an (n/m) x l matrix where the values are reduced (averaged) for the intervening states: + 1 2 3 4 1.5 3.5 + 5 6 7 8 -> 5.5 7.5 + 9 8 7 6 8.5 6.5 + 5 4 3 2 4.5 2.5 + + if m = 2 + + This process effectively resamples the data at a longer time span n units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining two columns are binned together as the last time period, while the first three are binned together. + + Input: + @param time_data n x l ndarray: measurements of an attribute at different time intervals + @param num_time_per_bin int: number of columns to average into a new column + Output: + ceil(n / m) x l ndarray of resampled time series + """ + + if time_data.shape[1] % num_time_per_bin == 0: + ## if fit is perfect, then use it + n_max = time_data.shape[1] / num_time_per_bin + else: + ## fit remainders into an additional column + n_max = time_data.shape[1] / num_time_per_bin + 1 + + return np.array([ + time_data[:, + num_time_per_bin*i:num_time_per_bin*(i+1)].mean(axis=1) + for i in range(n_max)]).T +def get_prob_dist(transition_matrix, lag_indices, unit_indices): + """ + given an array of transition matrices, look up the probability associated with the arrangements passed + + Input: + @param transition_matrix ndarray[k,k,k]: + @param lag_indices ndarray: + @param unit_indices ndarray: + + Output: + Array of probability distributions + """ + + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) + +def get_prob_stats(prob_dist, unit_indices): +# trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + + trend_up = np.array([prob_dist[:, i:].sum() for i in unit_indices]) + trend_down = np.array([prob_dist[:, :i].sum() for i in unit_indices]) + trend = trend_up - trend_down + volatility = prob_dist.std(axis=1) + + + return trend_up, trend_down, trend, volatility diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py new file mode 100644 index 0000000..dd7b8b0 --- /dev/null +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -0,0 +1,126 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.space_time_dynamics as std +import crankshaft.clustering as cc +from crankshaft import random_seeds +import json + +class SpaceTimeTests(unittest.TestCase): + """Testing class for Markov Functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T + + self.transition_matrix = p = np.array([ + [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], + [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], + [ 0. , 0.14 , 0.74 , 0.12 , 0. ], + [ 0. , 0.03571429, 0.32142857, 0.57142857, 0.07142857], + [ 0. , 0. , 0. , 0.16666667, 0.83333333]], + [[ 0.79831933, 0.16806723, 0.03361345, 0. , 0. ], + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0.00537634, 0.06989247, 0.8655914 , 0.05913978, 0. ], + [ 0. , 0. , 0.06372549, 0.90196078, 0.03431373], + [ 0. , 0. , 0. , 0.19444444, 0.80555556]], + [[ 0.84693878, 0.15306122, 0. , 0. , 0. ], + [ 0.08133971, 0.78947368, 0.1291866 , 0. , 0. ], + [ 0.00518135, 0.0984456 , 0.79274611, 0.0984456 , 0.00518135], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0. , 0. , 0. , 0.10204082, 0.89795918]], + [[ 0.8852459 , 0.09836066, 0. , 0.01639344, 0. ], + [ 0.03875969, 0.81395349, 0.13953488, 0. , 0.00775194], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0.02339181, 0.12865497, 0.75438596, 0.09356725], + [ 0. , 0. , 0. , 0.09661836, 0.90338164]], + [[ 0.33333333, 0.66666667, 0. , 0. , 0. ], + [ 0.0483871 , 0.77419355, 0.16129032, 0.01612903, 0. ], + [ 0.01149425, 0.16091954, 0.74712644, 0.08045977, 0. ], + [ 0. , 0.01036269, 0.06217617, 0.89637306, 0.03108808], + [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] + ) + + # def test_spatial_markov(self): + # """Test Spatial Markov.""" + # + # ans = "SELECT i.\"cartodb_id\" As id, " \ + # "i.\"dec_2013\"::numeric As attr1, " \ + # "i.\"jan_2014\"::numeric As attr2, " \ + # "i.\"feb_2014\"::numeric As attr3, " \ + # "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + # "FROM (SELECT * FROM a_list) As j " \ + # "WHERE j.\"dec_2013\" IS NOT NULL AND " \ + # "j.\"jan_2014\" IS NOT NULL AND " \ + # "j.\"feb_2014\" IS NOT NULL " \ + # "ORDER BY " \ + # "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + # "LIMIT 321 OFFSET 1 ) ) " \ + # "As neighbors " \ + # "FROM (SELECT * FROM a_list) As i " \ + # "WHERE i.\"dec_2013\" IS NOT NULL AND " \ + # "i.\"jan_2014\" IS NOT NULL AND " \ + # "i.\"feb_2014\" IS NOT NULL " \ + # "ORDER BY i.\"cartodb_id\" ASC;" + # + # subquery = self.params['subquery'] + # time_cols = self.params['time_cols'] + # num_time_per_bin = 1 + # permutations = 99 + # geom_col = self.params['geom_col'] + # id_col = self.params['id_col'] + # w_type = 'knn' + # num_ngbrs = self.params['num_ngbrs'] + # + # self.assertEqual(std.spatial_markov(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs), ans) + + def test_rebin_data(self): + """Test rebin_data""" + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## = 0.5, 2.5, 4.5, 6.5, 8.5 + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + for i in range(0, 10, 2)]).T + + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) + + ## sample in triple the time (uneven since 10 % 3 = 1): + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## = 1, 4, 7, 9 + ans_odd = np.array([i * np.ones(10, dtype=float) + for i in (1, 4, 7, 9)]).T + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) + def test_get_prob_dist(self): + """Test get_prob_dist""" + lag_indices = np.array([1, 2, 3, 4]) + unit_indices = np.array([1, 3, 2, 4]) + answer = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) + + self.assertTrue(np.array_equal(result, answer)) + + + From d4621a6e9c76071d1fd14acec571cef2449d124a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 14:50:52 -0400 Subject: [PATCH 029/183] adding passing tests --- .../crankshaft/space_time_dynamics/markov.py | 31 ++++++++--- .../test/test_space_time_dynamics.py | 55 +++++++++++++------ 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 911d558..db2be6f 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -54,7 +54,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge ## rebin time data if num_time_per_bin > 1: ## rebin - t_data = rebin_data(t_data, num_time_per_bin) + t_data = rebin_data(t_data, int(num_time_per_bin)) sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) @@ -68,7 +68,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) ## find the ups and down and overall distribution of each cell - trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist) ## output the results @@ -127,12 +127,29 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices): return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) def get_prob_stats(prob_dist, unit_indices): -# trend, trend_up, trend_down, volatility = get_prob_stats(prob_dist) + """ + get the statistics of the probability distributions - trend_up = np.array([prob_dist[:, i:].sum() for i in unit_indices]) - trend_down = np.array([prob_dist[:, :i].sum() for i in unit_indices]) - trend = trend_up - trend_down + Outputs: + @param trend_up ndarray(float): sum of probabilities for upward + movement (relative to the unit index of that prob) + @param trend_down ndarray(float): sum of probabilities for downard + movement (relative to the unit index of that prob) + @param trend ndarray(float): difference of upward and downward + movements + """ + + num_elements = len(prob_dist) + trend_up = np.empty(num_elements) + trend_down = np.empty(num_elements) + trend = np.empty(num_elements) + + for i in range(num_elements): + trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() + trend_down[i] = prob_dist[i, :unit_indices[i]].sum() + trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + + ## calculate volatility of distribution volatility = prob_dist.std(axis=1) - return trend_up, trend_down, trend, volatility diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py index dd7b8b0..c35aea5 100644 --- a/src/py/crankshaft/test/test_space_time_dynamics.py +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -28,9 +28,9 @@ class SpaceTimeTests(unittest.TestCase): "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) self.moran_data = json.loads(open(fixture_file('moran.json')).read()) - + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T - + self.transition_matrix = p = np.array([ [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], @@ -61,7 +61,7 @@ class SpaceTimeTests(unittest.TestCase): # def test_spatial_markov(self): # """Test Spatial Markov.""" - # + # # ans = "SELECT i.\"cartodb_id\" As id, " \ # "i.\"dec_2013\"::numeric As attr1, " \ # "i.\"jan_2014\"::numeric As attr2, " \ @@ -80,7 +80,7 @@ class SpaceTimeTests(unittest.TestCase): # "i.\"jan_2014\" IS NOT NULL AND " \ # "i.\"feb_2014\" IS NOT NULL " \ # "ORDER BY i.\"cartodb_id\" ASC;" - # + # # subquery = self.params['subquery'] # time_cols = self.params['time_cols'] # num_time_per_bin = 1 @@ -89,23 +89,23 @@ class SpaceTimeTests(unittest.TestCase): # id_col = self.params['id_col'] # w_type = 'knn' # num_ngbrs = self.params['num_ngbrs'] - # + # # self.assertEqual(std.spatial_markov(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs), ans) - + def test_rebin_data(self): """Test rebin_data""" - ## sample in double the time (even case since 10 % 2 = 0): - ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 ## = 0.5, 2.5, 4.5, 6.5, 8.5 - ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) for i in range(0, 10, 2)]).T - + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) ## sample in triple the time (uneven since 10 % 3 = 1): - ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 ## = 1, 4, 7, 9 - ans_odd = np.array([i * np.ones(10, dtype=float) + ans_odd = np.array([i * np.ones(10, dtype=float) for i in (1, 4, 7, 9)]).T self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) def test_get_prob_dist(self): @@ -119,8 +119,31 @@ class SpaceTimeTests(unittest.TestCase): [ 0. , 0. , 0. , 0.02352941, 0.97647059] ]) result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) - + self.assertTrue(np.array_equal(result, answer)) - - - + + def test_get_prob_stats(self): + """Test get_prob_stats""" + + probs = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + unit_indices = np.array([1, 3, 2, 4]) + answer_up = np.array([0.04245283, 0.03529412, 0.12376238, 0.]) + answer_down = np.array([0.0754717, 0.09411765, 0.0990099, 0.02352941]) + answer_trend = np.array([-0.03301887 / 0.88207547, -0.05882353 / 0.87058824, 0.02475248 / 0.77722772, -0.02352941 / 0.97647059]) + answer_volatility = np.array([ 0.34221495, 0.33705421, 0.29226542, 0.38834223]) + + result = std.get_prob_stats(probs, unit_indices) + result_up = result[0] + result_down = result[1] + result_trend = result[2] + result_volatility = result[3] + + self.assertTrue(np.allclose(result_up, answer_up)) + self.assertTrue(np.allclose(result_down, answer_down)) + self.assertTrue(np.allclose(result_trend, answer_trend)) + self.assertTrue(np.allclose(result_volatility, answer_volatility)) From dc0873cd2b004669f7f9496cd4ffb75a042251b7 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 15:33:54 -0400 Subject: [PATCH 030/183] add types to arrays --- src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index db2be6f..e809632 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -140,9 +140,9 @@ def get_prob_stats(prob_dist, unit_indices): """ num_elements = len(prob_dist) - trend_up = np.empty(num_elements) - trend_down = np.empty(num_elements) - trend = np.empty(num_elements) + trend_up = np.empty(num_elements, dtype=float) + trend_down = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) for i in range(num_elements): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() From bae2f04955989ccd570fe0e04f9522d663421bed Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 20:46:19 -0400 Subject: [PATCH 031/183] update signature used in plpython function --- src/pg/sql/11_markov.sql | 96 ++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 63a46ac..fa7c838 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -6,7 +6,7 @@ -- 2 | Pt2 | 11.0 | 13.2 | 12.5 -- ... -- Sample Function call: --- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', -- Array['date_1', 'date_2', 'date_3']) @@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION time_cols text[], num_time_per_bin int DEFAULT 1, permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', + geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs int DEFAULT 5) @@ -25,7 +25,7 @@ AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.space_time_predictions import spatial_markov # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) + return def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format @@ -34,48 +34,48 @@ $$ LANGUAGE plpythonu; -- 'date_1') -CREATE OR REPLACE FUNCTION - cdb_spatial_markov ( - subquery TEXT, - time_col_min text, - time_col_max text, - date_format text, -- '_YYYY_MM_DD' - num_time_per_bin INT DEFAULT 1, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) -$$ LANGUAGE plpythonu; - --- input table format: --- id | geom | date | measurement --- 1 | Pt1 | 12/3 | 13.2 --- 2 | Pt2 | 11/5 | 11.3 --- 3 | Pt1 | 11/13 | 12.9 --- 4 | Pt3 | 12/19 | 10.1 --- ... - -CREATE OR REPLACE FUNCTION - cdb_spatial_markov ( - subquery TEXT, - time_col text, - num_time_per_bin INT DEFAULT 1, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) -$$ LANGUAGE plpythonu; +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col_min text, +-- time_col_max text, +-- date_format text, -- '_YYYY_MM_DD' +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- +-- -- input table format: +-- -- id | geom | date | measurement +-- -- 1 | Pt1 | 12/3 | 13.2 +-- -- 2 | Pt2 | 11/5 | 11.3 +-- -- 3 | Pt1 | 11/13 | 12.9 +-- -- 4 | Pt3 | 12/19 | 10.1 +-- -- ... +-- +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col text, +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; From 9535506b939ae9f86be6e0b78cb11012504afb26 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 08:19:11 -0400 Subject: [PATCH 032/183] formated markov sql file a bit --- src/pg/sql/11_markov.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index fa7c838..f63af5d 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -23,9 +23,12 @@ CREATE OR REPLACE FUNCTION RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') - from crankshaft.space_time_predictions import spatial_markov - # TODO: use named parameters or a dictionary - return def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + + return spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format From 1de90a7d3944bb246a1f0200e6ee7c74008002ae Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 11:33:47 -0400 Subject: [PATCH 033/183] update output signature --- src/pg/sql/11_markov.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index f63af5d..09cca98 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -20,7 +20,7 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs int DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +RETURNS TABLE (trend numeric, trend_up numeric, trend_down numeric, volatility numeric, ids int) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') From cd3790860a2059a51c336ef6ca39fd3e5cc60235 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 24 Mar 2016 11:34:28 -0400 Subject: [PATCH 034/183] add working version --- .../crankshaft/space_time_dynamics/markov.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index e809632..60321e3 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -6,7 +6,7 @@ Spatial dynamics measurements using Spatial Markov import numpy as np import pysal as ps import plpy -from crankshaft.clustering import get_query +from crankshaft.clustering import get_query, get_weight def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): """ @@ -44,7 +44,9 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge try: query_result = plpy.execute(query) except: - zip([None],[None],[None]) + plpy.notice('** Query failed: %s' % query) + plpy.error('Query failed: check the input parameters') + return zip([None], [None], [None], [None], [None]) ## build weight weights = get_weight(query_result, w_type) @@ -58,17 +60,17 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) - ## get lags - lags = ps.lag_spatial(weights, t_data) + ## get lags of last time slice + lags = ps.lag_spatial(weights, t_data[:, -1]) ## get lag classes - lag_classes = ps.Quantiles(lags.flatten(), k=7).yb + lag_classes = ps.Quantiles(lags, k=7).yb ## look up probablity distribution for each unit according to class and lag class - prob_dist = get_prob_dist(lag_classes, sp_markov_result.classes) + prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) ## find the ups and down and overall distribution of each cell - trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist) + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1]) ## output the results @@ -78,7 +80,9 @@ def get_time_data(markov_data, time_cols): """ Extract the time columns and bin appropriately """ - return np.array([[x[t_col] for x in query_result] for t_col in time_cols], dtype=float) + num_attrs = len(time_cols) + return np.array([[x['attr' + str(i)] for x in markov_data] + for i in range(1, num_attrs+1)], dtype=float).T def rebin_data(time_data, num_time_per_bin): """ @@ -139,7 +143,7 @@ def get_prob_stats(prob_dist, unit_indices): movements """ - num_elements = len(prob_dist) + num_elements = len(unit_indices) trend_up = np.empty(num_elements, dtype=float) trend_down = np.empty(num_elements, dtype=float) trend = np.empty(num_elements, dtype=float) From e32bab3f88cc34e0543fb4bb5268495df4d73b34 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 16:10:34 -0400 Subject: [PATCH 035/183] removed pieces for pysal_utils --- src/py/crankshaft/test/test_clustering_moran.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 29c5bde..4002666 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -25,6 +25,11 @@ class MoranTest(unittest.TestCase): "subquery": "SELECT * FROM a_list", "geom_col": "the_geom", "num_ngbrs": 321} + self.params_markov = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) self.moran_data = json.loads(open(fixture_file('moran.json')).read()) From 369d1d2f41c7ce9d8755e4abf54d305123010572 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 25 Mar 2016 22:49:29 -0400 Subject: [PATCH 036/183] pylinting changes --- .../crankshaft/space_time_dynamics/markov.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 60321e3..db2425c 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -8,7 +8,8 @@ import pysal as ps import plpy from crankshaft.clustering import get_query, get_weight -def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): +def spatial_markov_trend(subquery, time_cols, num_time_per_bin, + permutations, geom_col, id_col, w_type, num_ngbrs): """ Predict the trends of a unit based on: 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) @@ -33,6 +34,9 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge @param """ + if num_time_per_bin < 1: + plpy.error('Error: number of time bins must be >= 1') + qvals = {"id_col": id_col, "time_cols": time_cols, "geom_col": geom_col, @@ -58,13 +62,14 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, ge ## rebin t_data = rebin_data(t_data, int(num_time_per_bin)) - sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, fixed=False) - - ## get lags of last time slice - lags = ps.lag_spatial(weights, t_data[:, -1]) + sp_markov_result = ps.Spatial_Markov(t_data, + weights, + k=7, + fixed=False, + permutations=permutations) ## get lag classes - lag_classes = ps.Quantiles(lags, k=7).yb + lag_classes = ps.Quantiles(ps.lag_spatial(weights, t_data[:, -1]), k=7).yb ## look up probablity distribution for each unit according to class and lag class prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) @@ -86,7 +91,8 @@ def get_time_data(markov_data, time_cols): def rebin_data(time_data, num_time_per_bin): """ - convert an n x l matrix into an (n/m) x l matrix where the values are reduced (averaged) for the intervening states: + convert an n x l matrix into an (n/m) x l matrix where the values are + reduced (averaged) for the intervening states: 1 2 3 4 1.5 3.5 5 6 7 8 -> 5.5 7.5 9 8 7 6 8.5 6.5 @@ -94,12 +100,17 @@ def rebin_data(time_data, num_time_per_bin): if m = 2 - This process effectively resamples the data at a longer time span n units longer than the input data. - For cases when there is a remainder (remainder(5/3) = 2), the remaining two columns are binned together as the last time period, while the first three are binned together. + This process effectively resamples the data at a longer time span n + units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining + two columns are binned together as the last time period, while the + first three are binned together. Input: - @param time_data n x l ndarray: measurements of an attribute at different time intervals - @param num_time_per_bin int: number of columns to average into a new column + @param time_data n x l ndarray: measurements of an attribute at + different time intervals + @param num_time_per_bin int: number of columns to average into a new + column Output: ceil(n / m) x l ndarray of resampled time series """ @@ -111,13 +122,12 @@ def rebin_data(time_data, num_time_per_bin): ## fit remainders into an additional column n_max = time_data.shape[1] / num_time_per_bin + 1 - return np.array([ - time_data[:, - num_time_per_bin*i:num_time_per_bin*(i+1)].mean(axis=1) + return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) for i in range(n_max)]).T def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ - given an array of transition matrices, look up the probability associated with the arrangements passed + given an array of transition matrices, look up the probability + associated with the arrangements passed Input: @param transition_matrix ndarray[k,k,k]: @@ -128,7 +138,8 @@ def get_prob_dist(transition_matrix, lag_indices, unit_indices): Array of probability distributions """ - return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] for i in range(len(lag_indices))]) + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] + for i in range(len(lag_indices))]) def get_prob_stats(prob_dist, unit_indices): """ @@ -144,9 +155,9 @@ def get_prob_stats(prob_dist, unit_indices): """ num_elements = len(unit_indices) - trend_up = np.empty(num_elements, dtype=float) + trend_up = np.empty(num_elements, dtype=float) trend_down = np.empty(num_elements, dtype=float) - trend = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) for i in range(num_elements): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() From 7695102500d8488ffda43f33a8eab67e6f45699f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 29 Mar 2016 11:24:57 -0700 Subject: [PATCH 037/183] update error message --- src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index db2425c..be7f999 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -49,7 +49,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, query_result = plpy.execute(query) except: plpy.notice('** Query failed: %s' % query) - plpy.error('Query failed: check the input parameters') + plpy.error('Spatial Markov failed: check the input parameters') return zip([None], [None], [None], [None], [None]) ## build weight @@ -91,7 +91,7 @@ def get_time_data(markov_data, time_cols): def rebin_data(time_data, num_time_per_bin): """ - convert an n x l matrix into an (n/m) x l matrix where the values are + Convert an n x l matrix into an (n/m) x l matrix where the values are reduced (averaged) for the intervening states: 1 2 3 4 1.5 3.5 5 6 7 8 -> 5.5 7.5 @@ -126,7 +126,7 @@ def rebin_data(time_data, num_time_per_bin): for i in range(n_max)]).T def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ - given an array of transition matrices, look up the probability + Given an array of transition matrices, look up the probability associated with the arrangements passed Input: From 6165d5e61ee5e594f9bf78ebd3ed001fb59011f2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 31 Mar 2016 09:27:37 -0400 Subject: [PATCH 038/183] restructure to accom list of time columns --- .../crankshaft/pysal_utils/pysal_utils.py | 58 ++++++++++++++----- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index fa06e26..f4def85 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -41,32 +41,62 @@ def query_attr_select(params): table name, etc.) """ - attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] - - template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " - attr_string = "" + template = "i.\"%(col)s\"::numeric As attr%(alias_num)s, " - for idx, val in enumerate(sorted(attrs)): - attr_string += template % {"col": val, "alias_num": idx + 1} + if 'time_cols' in params: + ## if markov analysis + attrs = params['time_cols'] + + for idx, val in enumerate(attrs): + attr_string += template % {"col": val, "alias_num": idx + 1} + else: + ## if moran's analysis + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')] + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": params[val], "alias_num": idx + 1} return attr_string def query_attr_where(params): """ + Construct where conditions when building neighbors query Create portion of WHERE clauses for weeding out NULL-valued geometries + Input: dict of params: + {'subquery': ..., + 'numerator': 'data1', + 'denominator': 'data2', + '': ...} + Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' + Input: + {'subquery': ..., + 'time_cols': ['time1', 'time2', 'time3'], + 'etc': ...} + Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT NULL AND idx_replace."time3" IS NOT NULL' """ - attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) - attr_string = [] + template = "idx_replace.\"%s\" IS NOT NULL" - for attr in attrs: - attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + if 'time_cols' in params: + ## markov where clauses + attrs = params['time_cols'] + # add values to template + for attr in attrs: + attr_string.append(template % attr) + else: + ## moran where clauses - if len(attrs) == 2: - attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + # get keys + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]) + # add values to template + for attr in attrs: + attr_string.append(template % params[attr]) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]]) out = " AND ".join(attr_string) From 314d1851db2b71c4526748ece273ba406434d1a0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 31 Mar 2016 09:29:51 -0400 Subject: [PATCH 039/183] fix tests for array-based inputs --- src/py/crankshaft/test/test_pysal_utils.py | 45 +++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 4ea0d9b..171fdbc 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -15,22 +15,38 @@ class PysalUtilsTest(unittest.TestCase): "geom_col": "the_geom", "num_ngbrs": 321} + self.params_array = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + def test_query_attr_select(self): """Test query_attr_select""" - ans = "i.\"{attr1}\"::numeric As attr1, " \ - "i.\"{attr2}\"::numeric As attr2, " + ans = "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " + + ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " self.assertEqual(pu.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params_array), ans_array) def test_query_attr_where(self): """Test pu.query_attr_where""" - ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ - "idx_replace.\"{attr2}\" IS NOT NULL AND " \ - "idx_replace.\"{attr2}\" <> 0" + ans = "idx_replace.\"andy\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" <> 0" + + ans_array = "idx_replace.\"_2013_dec\" IS NOT NULL AND " \ + "idx_replace.\"_2014_jan\" IS NOT NULL AND " \ + "idx_replace.\"_2014_feb\" IS NOT NULL" self.assertEqual(pu.query_attr_where(self.params), ans) + self.assertEqual(pu.query_attr_where(self.params_array), ans_array) def test_knn(self): """Test knn neighbors constructor""" @@ -53,8 +69,27 @@ class PysalUtilsTest(unittest.TestCase): "i.\"jay_z\" IS NOT NULL AND " \ "i.\"jay_z\" <> 0 " \ "ORDER BY i.\"cartodb_id\" ASC;" + + ans_array = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"_2013_dec\" IS NOT NULL AND " \ + "j.\"_2014_jan\" IS NOT NULL AND " \ + "j.\"_2014_feb\" IS NOT NULL " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"_2013_dec\" IS NOT NULL AND " \ + "i.\"_2014_jan\" IS NOT NULL AND " \ + "i.\"_2014_feb\" IS NOT NULL "\ + "ORDER BY i.\"cartodb_id\" ASC;" self.assertEqual(pu.knn(self.params), ans) + self.assertEqual(pu.knn(self.params_array), ans_array) def test_queen(self): """Test queen neighbors constructor""" From c18baf26d821594629f9ec5f669ee61e556497a6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 31 Mar 2016 09:35:10 -0400 Subject: [PATCH 040/183] adding module refs for pysaul utils --- .../crankshaft/space_time_dynamics/markov.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index be7f999..da7271c 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -6,7 +6,7 @@ Spatial dynamics measurements using Spatial Markov import numpy as np import pysal as ps import plpy -from crankshaft.clustering import get_query, get_weight +import crankshaft.pysal_utils as pu def spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs): @@ -43,7 +43,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, "subquery": subquery, "num_ngbrs": num_ngbrs} - query = get_query(w_type, qvals) + query = pu.construct_neighbor_query(w_type, qvals) try: query_result = plpy.execute(query) @@ -53,7 +53,7 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, return zip([None], [None], [None], [None], [None]) ## build weight - weights = get_weight(query_result, w_type) + weights = pu.get_weight(query_result, w_type) ## prep time data t_data = get_time_data(query_result, time_cols) @@ -81,6 +81,14 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, return zip(trend, trend_up, trend_down, volatility, weights.id_order) +def spatial_markov_predict(subquery, time_cols, num_time_per_bin, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Filler for this future function + """ + + return None + def get_time_data(markov_data, time_cols): """ Extract the time columns and bin appropriately @@ -98,13 +106,13 @@ def rebin_data(time_data, num_time_per_bin): 9 8 7 6 8.5 6.5 5 4 3 2 4.5 2.5 - if m = 2 + if m = 2, the 4 x 4 matrix is transformed to a 2 x 4 matrix. This process effectively resamples the data at a longer time span n units longer than the input data. For cases when there is a remainder (remainder(5/3) = 2), the remaining two columns are binned together as the last time period, while the - first three are binned together. + first three are binned together for the first period. Input: @param time_data n x l ndarray: measurements of an attribute at From e73862a6e1bbc20957d971dba914fab228d62b12 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 31 Mar 2016 11:25:30 -0400 Subject: [PATCH 041/183] adding function to predict the importance of different features to a dataset. --- pg/sql/0.0.1/05_segmentation.sql | 12 +++++ .../crankshaft/segmentation/segmentation.py | 48 +++++++++++-------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/pg/sql/0.0.1/05_segmentation.sql b/pg/sql/0.0.1/05_segmentation.sql index 8cbf1c9..b9bca6a 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/pg/sql/0.0.1/05_segmentation.sql @@ -13,6 +13,18 @@ AS $$ return segmentation.create_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') $$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION + cdb_correlated_variables( + query text, + geoid_column text DEFAULT 'geoid', + census_table text DEFAULT 'ml_learning_block_groups_clipped' + ) +RETURNS TABLE(feature text, importance NUMERIC, std NUMERIC) +AS $$ + from crankshaft.segmentation import correlated_variables + return correlated_variables(query,geoid_column,census_table) +$$ LANGUAGE plpythonu; + CREATE OR REPLACE FUNCTION cdb_predict_segment ( segment_name TEXT, diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/python/crankshaft/crankshaft/segmentation/segmentation.py index 130c8e4..0a1a8da 100644 --- a/python/crankshaft/crankshaft/segmentation/segmentation.py +++ b/python/crankshaft/crankshaft/segmentation/segmentation.py @@ -30,6 +30,20 @@ def create_segment(segment_name,table_name,column_name,geoid_column,census_table # predict_segment return accuracy +def correlated_variables(query,geoid_column,census_table): + """ + returns the columns which are importaint for the random forrest model + """ + data = pd.DataFrame(join_with_census(query,geoid_column, census_table)) + features = data[data.columns.difference(['target', 'the_geom_webmercator', 'geoid','the_geom'])] + target, mean, std = normalize(data['target']) + model, accuracy, used_features = train_model(target,features, test_split=0.2) + std = np.std([tree.feature_importances_ for tree in model.estimators_], + axis=0) + importances = model.feature_importances_ + return zip(features,importances,std) + + def create_and_predict_segment(segment_name,query,geoid_column,census_table,target_table,method): """ generate a segment with machine learning @@ -98,15 +112,6 @@ def join_with_census(query, geoid_column, census_table): def query_to_dictionary(result): return [ dict(zip(r.keys(), r.values())) for r in result ] -def query_in_batches(query,batch_size): - cursor = plpy.cursor(query) - while True: - rows = cursor.fetch(batch_size) - if not rows: - break - else: - yield query_to_dictionary(rows) - def predict_segment(model,features,geoid_column,census_table): """ predict a segment with machine learning @@ -117,21 +122,22 @@ def predict_segment(model,features,geoid_column,census_table): # features = ",".join(features) joined_features = ','.join(['\"'+a+'\"::numeric' for a in features]) - targets = pd.DataFrame(query_to_dictionary(plpy.execute('select {joined_features} from {census_table}'.format(**locals())))) + = plpy.execute() + cursor = plpy.cursor('select {joined_features} from {census_table}'.format(**locals())) + results = [] + while True: + rows = cursor.fetch(batch_size) - predition = [] - for batch in query_in_batches('select {joined_features} from {census_table}'.format(**locals()),2000): - targets = pd.DataFrame(batch) - plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) - plpy.notice(joined_features) - targets = targets.dropna(axis =1, how='all').fillna(0) - plpy.notice('predicting:' + str(len(features)) + ' '+str(np.shape(targets))) - batch_prediction = model.predict(targets) - prediciton.append(batch_prediction.to_maxtrix) + if not rows: + break - geo_ids = plpy.execute('select geoid from {census_table}'.format(**locals())) + batch = pd.DataFrame(query_to_dictionary(rows)) + batch_features = batch.dropna(axis =1, how='all').fillna(0) + prediction = model.predict(batch_features) + results.append(prediction) + plpy.notice('predicting: predicted') - return [[a['geoid'] for a in geo_ids],prediction] + return [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction def fetch_model(model_name): From 693f6a68db78379dddd14a27733951b7f825d579 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 31 Mar 2016 11:26:33 -0400 Subject: [PATCH 042/183] pylint suggested changes --- .../crankshaft/space_time_dynamics/markov.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index da7271c..e42176e 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -47,8 +47,8 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, try: query_result = plpy.execute(query) - except: - plpy.notice('** Query failed: %s' % query) + except plpy.SPIError, err: + plpy.notice('** Query failed with exception %s: %s' % (err, query)) plpy.error('Spatial Markov failed: check the input parameters') return zip([None], [None], [None], [None], [None]) @@ -75,20 +75,13 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) ## find the ups and down and overall distribution of each cell - trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1]) + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, + sp_markov_result.classes[:, -1]) ## output the results return zip(trend, trend_up, trend_down, volatility, weights.id_order) -def spatial_markov_predict(subquery, time_cols, num_time_per_bin, - permutations, geom_col, id_col, w_type, num_ngbrs): - """ - Filler for this future function - """ - - return None - def get_time_data(markov_data, time_cols): """ Extract the time columns and bin appropriately @@ -131,7 +124,7 @@ def rebin_data(time_data, num_time_per_bin): n_max = time_data.shape[1] / num_time_per_bin + 1 return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) - for i in range(n_max)]).T + for i in range(n_max)]).T def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ Given an array of transition matrices, look up the probability From 3294eb35abb7e8166c4b672c56510b0a220a90d1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 1 Apr 2016 08:22:27 -0400 Subject: [PATCH 043/183] remove if conditions and relying on pysal.W to build weights norms --- .../crankshaft/pysal_utils/pysal_utils.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index f4def85..9d09642 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -20,19 +20,23 @@ def construct_neighbor_query(w_type, query_vals): def get_weight(query_res, w_type='knn', num_ngbrs=5): """ Construct PySAL weight from return value of query - @param query_res: query results with attributes and neighbors + @param query_res dict-like: query results with attributes and neighbors """ - if w_type == 'knn': - row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs - weights = {x['id']: row_normed_weights for x in query_res} - else: - weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - if len(x['neighbors']) > 0 - else [] for x in query_res} + # if w_type == 'knn': + # row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + # weights = {x['id']: row_normed_weights for x in query_res} + # else: + # weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + # if len(x['neighbors']) > 0 + # else [] for x in query_res} neighbors = {x['id']: x['neighbors'] for x in query_res} + print 'len of neighbors: %d' % len(neighbors) - return ps.W(neighbors, weights) + built_weight = ps.W(neighbors) + built_weight.transform = 'r' + + return built_weight def query_attr_select(params): """ From ef475adc26a51642af64496e60a19c343781da1c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 1 Apr 2016 08:23:10 -0400 Subject: [PATCH 044/183] testing array outputs --- src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index e42176e..1600583 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -62,6 +62,11 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, ## rebin t_data = rebin_data(t_data, int(num_time_per_bin)) + print 'shape of t_data %d, %d' % t_data.shape + print 'number of weight objects: %d, %d' % (weights.sparse).shape + print 'first num elements: %f' % t_data[0, 0] + # ls = ps.lag_spatial(weights, t_data) + sp_markov_result = ps.Spatial_Markov(t_data, weights, k=7, @@ -88,7 +93,7 @@ def get_time_data(markov_data, time_cols): """ num_attrs = len(time_cols) return np.array([[x['attr' + str(i)] for x in markov_data] - for i in range(1, num_attrs+1)], dtype=float).T + for i in range(1, num_attrs+1)], dtype=float).transpose() def rebin_data(time_data, num_time_per_bin): """ From 9ba9d07bb503a6c2e741043f2d00561083c55829 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 1 Apr 2016 08:23:46 -0400 Subject: [PATCH 045/183] building tests for markov --- .../test/test_space_time_dynamics.py | 59 ++++++++----------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py index ca1a7f4..819852b 100644 --- a/src/py/crankshaft/test/test_space_time_dynamics.py +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -12,7 +12,6 @@ import unittest from helper import plpy, fixture_file import crankshaft.space_time_dynamics as std -import crankshaft.clustering as cc from crankshaft import random_seeds import json @@ -26,8 +25,8 @@ class SpaceTimeTests(unittest.TestCase): "subquery": "SELECT * FROM a_list", "geom_col": "the_geom", "num_ngbrs": 321} - self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) - self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + self.neighbors_data = json.loads(open(fixture_file('neighbors_markov.json')).read()) + # self.moran_data = json.loads(open(fixture_file('markov.json')).read()) self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T @@ -59,38 +58,28 @@ class SpaceTimeTests(unittest.TestCase): [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] ) - # def test_spatial_markov(self): - # """Test Spatial Markov.""" - # - # ans = "SELECT i.\"cartodb_id\" As id, " \ - # "i.\"dec_2013\"::numeric As attr1, " \ - # "i.\"jan_2014\"::numeric As attr2, " \ - # "i.\"feb_2014\"::numeric As attr3, " \ - # "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - # "FROM (SELECT * FROM a_list) As j " \ - # "WHERE j.\"dec_2013\" IS NOT NULL AND " \ - # "j.\"jan_2014\" IS NOT NULL AND " \ - # "j.\"feb_2014\" IS NOT NULL " \ - # "ORDER BY " \ - # "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ - # "LIMIT 321 OFFSET 1 ) ) " \ - # "As neighbors " \ - # "FROM (SELECT * FROM a_list) As i " \ - # "WHERE i.\"dec_2013\" IS NOT NULL AND " \ - # "i.\"jan_2014\" IS NOT NULL AND " \ - # "i.\"feb_2014\" IS NOT NULL " \ - # "ORDER BY i.\"cartodb_id\" ASC;" - # - # subquery = self.params['subquery'] - # time_cols = self.params['time_cols'] - # num_time_per_bin = 1 - # permutations = 99 - # geom_col = self.params['geom_col'] - # id_col = self.params['id_col'] - # w_type = 'knn' - # num_ngbrs = self.params['num_ngbrs'] - # - # self.assertEqual(std.spatial_markov(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs), ans) + def test_spatial_markov(self): + """Test Spatial Markov.""" + data = [ { 'id': d['id'], + 'attr1': d['y1929'], + 'attr2': d['y1930'], + 'attr3': d['y1931'], + 'neighbors': d['neighbors'] } for d in self.neighbors_data] + + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + + result = std.spatial_markov_trend('subquery', ['y1929', 'y1930', 'y1931', 'y1932', 'y1933', 'y1934', 'y1935', 'y1936', 'y1937', 'y1938', 'y1939'], 1, 99, 'the_geom', 'cartodb_id', 'knn', 5) + + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + print result[0] + assertTrue(result[0] == None) + # expected = self.moran_data + # for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + # self.assertAlmostEqual(res_val, exp_val) + + def test_rebin_data(self): """Test rebin_data""" From 95247f66bbf5c20188f07e0b5fed780075c933f7 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 1 Apr 2016 08:24:09 -0400 Subject: [PATCH 046/183] updating test info --- src/py/crankshaft/test/test_clustering_moran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 4002666..1866692 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -81,7 +81,7 @@ class MoranTest(unittest.TestCase): data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1235) - result = cc.moran('table', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = cc.moran('subquery', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) print 'result == None?', result == None result_moran = result[0][0] expected_moran = np.array([row[0] for row in self.moran_data]).mean() From c44434ef0885b0be729a67b867e75dfc8f55c4e3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 1 Apr 2016 08:24:33 -0400 Subject: [PATCH 047/183] adding fixture for markov testing --- src/py/crankshaft/test/fixtures/neighbors_markov.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/py/crankshaft/test/fixtures/neighbors_markov.json diff --git a/src/py/crankshaft/test/fixtures/neighbors_markov.json b/src/py/crankshaft/test/fixtures/neighbors_markov.json new file mode 100644 index 0000000..d83122a --- /dev/null +++ b/src/py/crankshaft/test/fixtures/neighbors_markov.json @@ -0,0 +1 @@ +[{"neighbors": [19, 27, 30], "y1937": 0.93697174595508337, "y1936": 0.9711757377894239, "y1935": 0.96227785579390823, "y1934": 1.0133392128761989, "y1933": 1.0306187269724305, "y1932": 1.044349070100143, "y1931": 1.0385721458894417, "y1930": 1.0588235294117647, "y1929": 1.0305452082627835, "id": 43, "y1939": 0.96816333237481, "y1938": 0.95423699321385069}, {"neighbors": [15, 18, 31, 40, 46], "y1937": 0.81719391451340251, "y1936": 0.8041582542205421, "y1935": 0.81351992639581616, "y1934": 0.84665417263807741, "y1933": 0.86901283191462331, "y1932": 0.81258941344778257, "y1931": 0.81069977632720136, "y1930": 0.70588235294117652, "y1929": 0.7054520826278361, "id": 44, "y1939": 0.83999507045146449, "y1938": 0.81433791543413947}, {"neighbors": [5, 13, 14, 23, 39, 48], "y1937": 0.80173871045641154, "y1936": 0.81652991967008892, "y1935": 0.95065614255968234, "y1934": 0.68526072097894397, "y1933": 0.83852115360182955, "y1932": 0.87839771101573672, "y1931": 0.90491623681928157, "y1930": 0.95772058823529416, "y1929": 0.96877751439214355, "id": 25, "y1939": 0.78872776568212632, "y1938": 0.84565860448929875}, {"neighbors": [2, 4, 10, 35, 42], "y1937": 1.4721081864283989, "y1936": 1.7382189956613256, "y1935": 1.5294174616241343, "y1934": 1.4446036820637196, "y1933": 1.5093380764832931, "y1932": 1.5736766809728182, "y1931": 1.4285844707171225, "y1930": 1.53125, "y1929": 1.4109041652556722, "id": 26, "y1939": 1.6977365156307769, "y1938": 1.6286758308682789}, {"neighbors": [17, 19, 43], "y1937": 1.091523786524994, "y1936": 1.1072640577344388, "y1935": 1.1575226381289041, "y1934": 1.2593980817991401, "y1933": 1.2684538178122222, "y1932": 1.2217453505007152, "y1931": 1.2226229059204821, "y1930": 1.1893382352941178, "y1929": 1.11506942092787, "id": 27, "y1939": 1.1061906913691821, "y1938": 1.1129284844266574}, {"neighbors": [7, 30, 36], "y1937": 1.4431296788215406, "y1936": 1.4619184672881136, "y1935": 1.4527141542782431, "y1934": 1.5160401278800575, "y1933": 1.5947147757591158, "y1932": 1.6795422031473533, "y1931": 1.6126352307481628, "y1930": 1.556985294117647, "y1929": 1.4921774466644089, "id": 28, "y1939": 1.4768927412397814, "y1938": 1.4553680180963982}, {"neighbors": [13, 32, 39, 47], "y1937": 1.0432262738468969, "y1936": 0.97323768203101502, "y1935": 1.0482785337271803, "y1934": 0.94719435563884913, "y1933": 0.93914369203404913, "y1932": 1.03862660944206, "y1931": 1.0013237777879218, "y1930": 1.0147058823529411, "y1929": 0.97365391127666778, "id": 21, "y1939": 1.0194306371441482, "y1938": 1.0314946928832434}, {"neighbors": [1, 3, 16, 40], "y1937": 0.43274571359574981, "y1936": 0.47218523132436957, "y1935": 0.41140864849159847, "y1934": 0.46036820637195458, "y1933": 0.3994409858975988, "y1932": 0.36337625178826893, "y1931": 0.38343908339800065, "y1930": 0.37132352941176472, "y1929": 0.46488316965797494, "id": 22, "y1939": 0.40422297991208972, "y1938": 0.41969723333913345}, {"neighbors": [3, 11, 13, 14, 15, 25, 34, 40], "y1937": 0.98140545761893261, "y1936": 0.9608660165814682, "y1935": 0.97622391167497935, "y1934": 0.97100650424429502, "y1933": 1.0184220556473129, "y1932": 1.044349070100143, "y1931": 1.0758205139909618, "y1930": 1.03125, "y1929": 1.0094141550965119, "id": 23, "y1939": 0.9937969847594792, "y1938": 0.99182182008004172}, {"neighbors": [10, 32, 39, 48], "y1937": 0.98913305964742815, "y1936": 0.97942351475578848, "y1935": 1.1063870998983099, "y1934": 0.96306912137581313, "y1933": 0.90865201372125526, "y1932": 0.96995708154506433, "y1931": 0.83699274204592145, "y1930": 0.92095588235294112, "y1929": 0.96227565187944453, "id": 24, "y1939": 1.0509797477714333, "y1938": 1.0795197494344875}, {"neighbors": [11, 13, 20, 21], "y1937": 1.0644771794252597, "y1936": 1.0680871171442072, "y1935": 1.0715219601956321, "y1934": 1.005401830007717, "y1933": 1.0153728878160335, "y1932": 1.0357653791130186, "y1931": 1.0276167435066417, "y1930": 1.0808823529411764, "y1929": 1.0939383677615984, "id": 47, "y1939": 1.0115433594873271, "y1938": 1.0586392900643813}, {"neighbors": [5, 10, 24, 25, 39, 42], "y1937": 1.172663607824197, "y1936": 1.1361312771167147, "y1935": 1.1528739528352137, "y1934": 1.0874214529820307, "y1933": 1.13124126540465, "y1932": 1.0701001430615165, "y1931": 1.0429543068425617, "y1930": 1.0753676470588236, "y1929": 1.0971892990179477, "id": 48, "y1939": 1.1574579961385203, "y1938": 1.1713937706629545}, {"neighbors": [10, 35], "y1937": 1.1572084037672059, "y1936": 1.173246273465355, "y1935": 1.1389278969541425, "y1934": 1.1720868702458385, "y1933": 1.146487104561047, "y1932": 1.150214592274678, "y1931": 1.1700369744830419, "y1930": 1.2095588235294117, "y1929": 1.2044700304774805, "id": 45, "y1939": 1.2106971203220638, "y1938": 1.2152427353401773}, {"neighbors": [15, 18, 33, 36, 44], "y1937": 0.8075344119777832, "y1936": 0.8041582542205421, "y1935": 0.7833034719868287, "y1934": 0.82813361261161944, "y1933": 0.78973446830135952, "y1932": 0.73533619456366239, "y1931": 0.78002464965536134, "y1930": 0.75, "y1929": 0.74771418896037922, "id": 46, "y1939": 0.76506593271166257, "y1938": 0.77257699669392721}, {"neighbors": [2, 5, 34, 41, 42], "y1937": 0.69934798357884564, "y1936": 0.70724687486575877, "y1935": 0.67870805287879521, "y1934": 0.65351118950501597, "y1933": 0.64337441239994919, "y1932": 0.59513590844062947, "y1931": 0.63322225772584106, "y1930": 0.61397058823529416, "y1929": 0.66644090755164231, "id": 29, "y1939": 0.70393953087129768, "y1938": 0.70575952670958764}, {"neighbors": [6, 19, 28, 36, 43], "y1937": 1.6189326249698142, "y1936": 1.666050947205636, "y1935": 1.6781753910222263, "y1934": 1.7991401168559145, "y1933": 1.908779062380892, "y1932": 1.9341917024320459, "y1931": 1.9303418998493633, "y1930": 1.9025735294117647, "y1929": 1.8725364036572976, "id": 30, "y1939": 1.6267510167193855, "y1938": 1.6474682443013746}, {"neighbors": [3, 16, 29, 34], "y1937": 0.8075344119777832, "y1936": 0.76704325787190164, "y1935": 0.75773570287153158, "y1934": 0.77786352111123369, "y1933": 0.78363613263880072, "y1932": 0.76108726752503575, "y1931": 0.76249600584288135, "y1930": 0.75735294117647056, "y1929": 0.7785980358956992, "id": 41, "y1939": 0.8222486957236167, "y1938": 0.84357055855228813}, {"neighbors": [2, 5, 10, 26, 29, 48], "y1937": 0.85776382516300409, "y1936": 0.95468018385669484, "y1935": 0.90416928962277854, "y1934": 0.82019622974313755, "y1933": 0.90865201372125526, "y1932": 0.87267525035765381, "y1931": 0.80850869585064133, "y1930": 0.9154411764705882, "y1929": 0.89563156112428033, "id": 42, "y1939": 0.90309329170603458, "y1938": 0.92709239603271265}, {"neighbors": [4, 5, 26, 29, 42], "y1937": 0.97367785559043707, "y1936": 0.95261823961510372, "y1935": 0.96692654108759857, "y1934": 0.95777753279682509, "y1933": 0.93914369203404913, "y1932": 0.91845493562231761, "y1931": 0.93997352444424165, "y1930": 0.95588235294117652, "y1929": 0.97527937690484245, "id": 2, "y1939": 0.96619151296060468, "y1938": 0.99808595789107357}, {"neighbors": [8, 9, 22, 40], "y1937": 0.51581743540207681, "y1936": 0.51754800463937456, "y1935": 0.50438235436540602, "y1934": 0.55826259508323228, "y1933": 0.50616185999237706, "y1932": 0.46351931330472101, "y1931": 0.49080202674944085, "y1930": 0.49080882352941174, "y1929": 0.52502539790044023, "id": 1, "y1939": 0.4968984923797396, "y1938": 0.50948320863058982}, {"neighbors": [2, 26, 35], "y1937": 1.5358609031634871, "y1936": 1.589759010266764, "y1935": 1.5340661469178247, "y1934": 1.5954139565648771, "y1933": 1.6648456358785415, "y1932": 1.659513590844063, "y1931": 1.6411192769434428, "y1930": 1.630514705882353, "y1929": 1.6108364375211648, "id": 4, "y1939": 1.5399909624943515, "y1938": 1.6098834174351835}, {"neighbors": [16, 22, 23, 34, 40, 41], "y1937": 0.49456652982371407, "y1936": 0.50930022767300998, "y1935": 0.48113892789695412, "y1934": 0.49476353213537649, "y1933": 0.4787193495108627, "y1932": 0.44921316165951358, "y1931": 0.47108230246040078, "y1930": 0.41911764705882354, "y1929": 0.50389434473416861, "id": 3, "y1939": 0.49098303413712363, "y1938": 0.48233861144945189}, {"neighbors": [19, 30, 37], "y1937": 1.6614344361265394, "y1936": 1.6619270587224537, "y1935": 1.6409859086727034, "y1934": 1.7277036710395768, "y1933": 1.7776648456358786, "y1932": 1.7739628040057225, "y1931": 1.7550554617245631, "y1930": 1.693014705882353, "y1929": 1.6644768032509312, "id": 6, "y1939": 1.648441030275644, "y1938": 1.6057073255611622}, {"neighbors": [2, 14, 25, 29, 34, 42, 48], "y1937": 1.0277710697899058, "y1936": 1.1175737789423943, "y1935": 1.0320081351992638, "y1934": 0.9736522985337891, "y1933": 1.0763562444416213, "y1932": 1.0128755364806867, "y1931": 1.0319989044597617, "y1930": 1.0625, "y1929": 1.0305452082627835, "id": 5, "y1939": 1.0174588177299428, "y1938": 1.0565512441273708}, {"neighbors": [1, 9], "y1937": 0.94083554696933103, "y1936": 0.92787490871601008, "y1935": 0.87395283521379108, "y1934": 0.92073641274390916, "y1933": 0.8781603354084615, "y1932": 0.9127324749642346, "y1931": 0.87205002967088152, "y1930": 0.86397058823529416, "y1929": 0.84199119539451395, "id": 8, "y1939": 0.97605061003163129, "y1938": 0.96050113102488255}, {"neighbors": [18, 28, 36], "y1937": 1.8333735812605652, "y1936": 1.7897676017011039, "y1935": 1.6293641954384774, "y1934": 1.7065373167236249, "y1933": 1.7197306568415704, "y1932": 1.6881258941344779, "y1931": 1.6980873693340028, "y1930": 1.5753676470588236, "y1929": 1.677480528276329, "id": 7, "y1939": 1.7726656533705789, "y1938": 1.6599965199234383}, {"neighbors": [24, 26, 35, 42, 45, 48], "y1937": 0.81719391451340251, "y1936": 0.97942351475578848, "y1935": 0.92741271609123044, "y1934": 1.0662550986660788, "y1933": 0.69216109770041934, "y1932": 0.78397711015736771, "y1931": 0.81946409823344135, "y1930": 0.92463235294117652, "y1929": 0.82411107348459189, "id": 10, "y1939": 0.86168508400772303, "y1938": 0.88950756916652163}, {"neighbors": [1, 8, 31, 38, 40], "y1937": 0.6046848587297754, "y1936": 0.62270716096052237, "y1935": 0.62292382935451063, "y1934": 0.64557380663653408, "y1933": 0.62203023758099352, "y1932": 0.57224606580829762, "y1931": 0.560916601999361, "y1930": 0.56433823529411764, "y1929": 0.5640365729766339, "id": 9, "y1939": 0.60929219898944253, "y1938": 0.60553332173307806}, {"neighbors": [1, 3, 9, 15, 22, 23, 31, 44], "y1937": 0.64525476937937698, "y1936": 0.6268310494437046, "y1935": 0.61362645876712985, "y1934": 0.64821960092602804, "y1933": 0.62203023758099352, "y1932": 0.5665236051502146, "y1931": 0.60692929200712109, "y1930": 0.59742647058823528, "y1929": 0.61442600745005072, "id": 40, "y1939": 0.61323583781785318, "y1938": 0.62641378110318424}, {"neighbors": [13, 21, 24, 25, 32, 48], "y1937": 0.62400386380101425, "y1936": 0.50311439494823662, "y1935": 0.71822187787516345, "y1934": 0.48682614926689455, "y1933": 0.39334265023504006, "y1932": 0.54077253218884125, "y1931": 0.52805039485096095, "y1930": 0.67279411764705888, "y1929": 0.69244835760243817, "id": 39, "y1939": 0.68027769790083392, "y1938": 0.6681746998433965}, {"neighbors": [11, 15, 20, 33], "y1937": 1.0567495773967641, "y1936": 0.99179518020533519, "y1935": 0.97854825432182457, "y1934": 0.94984014992834309, "y1933": 0.89645534239613778, "y1932": 0.88698140200286124, "y1931": 0.95969324873328166, "y1930": 0.94485294117647056, "y1929": 0.9866576363020656, "id": 12, "y1939": 1.0214024565583535, "y1938": 0.98555768226900986}, {"neighbors": [12, 13, 15, 23, 47], "y1937": 1.4122192707075585, "y1936": 1.3402637570342368, "y1935": 1.3318483366422933, "y1934": 1.336126116194466, "y1933": 1.3324863422690891, "y1932": 1.390557939914163, "y1931": 1.4702149997717626, "y1930": 1.4834558823529411, "y1929": 1.540941415509651, "id": 11, "y1939": 1.3881608676005424, "y1938": 1.353053767182878}, {"neighbors": [5, 23, 25, 34], "y1937": 0.82685341704902193, "y1936": 0.79797242149576864, "y1935": 0.8414120381579584, "y1934": 0.75934296108477572, "y1933": 0.76229195781984505, "y1932": 0.76108726752503575, "y1931": 0.87862327110056149, "y1930": 0.85845588235294112, "y1929": 0.86474771418896035, "id": 14, "y1939": 0.75323501622643063, "y1938": 0.79972159387506525}, {"neighbors": [11, 21, 23, 25, 39, 47], "y1937": 1.0103839652257909, "y1936": 0.81034408694531546, "y1935": 0.98784562490920536, "y1934": 0.71171866387388383, "y1933": 0.77143946131368324, "y1932": 0.84978540772532185, "y1931": 0.87643219062400146, "y1930": 0.9375, "y1929": 0.94439552996952247, "id": 13, "y1939": 0.93661422174752496, "y1938": 0.95632503915086131}, {"neighbors": [3, 22, 41], "y1937": 0.68196087901473079, "y1936": 0.68044159972507412, "y1935": 0.67405936758510476, "y1934": 0.70113548671590786, "y1933": 0.69216109770041934, "y1932": 0.68955650929899859, "y1931": 0.69676359154608114, "y1930": 0.65257352941176472, "y1929": 0.67294277006434133, "id": 16, "y1939": 0.70985498911391365, "y1938": 0.72663998607969371}, {"neighbors": [11, 12, 23, 33, 40, 44, 46], "y1937": 0.65877807292924417, "y1936": 0.60621160702779331, "y1935": 0.61595080141397507, "y1934": 0.61647006945210014, "y1933": 0.62507940541227291, "y1932": 0.60371959942775388, "y1931": 0.63760441867896112, "y1930": 0.59742647058823528, "y1929": 0.63880799187267179, "id": 15, "y1939": 0.60140492133262136, "y1938": 0.62014964329215239}, {"neighbors": [7, 36, 44, 46], "y1937": 1.2847138372373823, "y1936": 1.2742815413033206, "y1935": 1.2737397704711635, "y1934": 1.3837504134053578, "y1933": 1.4209122093761912, "y1932": 1.4649499284692418, "y1931": 1.3979093440452823, "y1930": 1.3088235294117647, "y1929": 1.2483576024381984, "id": 18, "y1939": 1.3073162716181244, "y1938": 1.3217330781277188}, {"neighbors": [27], "y1937": 0.98526925863318038, "y1936": 1.0433437862451136, "y1935": 0.99946733814343125, "y1934": 1.1006504244295006, "y1933": 1.13124126540465, "y1932": 1.0786838340486409, "y1931": 1.0758205139909618, "y1930": 1.0588235294117647, "y1929": 0.97690484253301724, "id": 17, "y1939": 0.97605061003163129, "y1938": 0.98346963633199924}, {"neighbors": [12, 33, 47], "y1937": 1.32335184737986, "y1936": 1.2763434855449116, "y1935": 1.2319016028279501, "y1934": 1.1985448131407783, "y1933": 1.0580612374539449, "y1932": 1.1273247496423462, "y1931": 1.183183457342402, "y1930": 1.2077205882352942, "y1929": 1.2841178462580425, "id": 20, "y1939": 1.2323871338783223, "y1938": 1.1943622759700714}, {"neighbors": [6, 27, 30, 37, 43], "y1937": 1.4141511712146824, "y1936": 1.4722281884960693, "y1935": 1.4945523219214565, "y1934": 1.6112887223018411, "y1933": 1.7044848176851735, "y1932": 1.753934191702432, "y1931": 1.6630300817090429, "y1930": 1.536764705882353, "y1929": 1.4726718591263122, "id": 19, "y1939": 1.4275972558846486, "y1938": 1.4031668696711328}, {"neighbors": [21, 24, 39], "y1937": 0.6297995653223859, "y1936": 0.48249495253232527, "y1935": 0.63222119994189141, "y1934": 0.47624297210891858, "y1933": 0.44517850336678949, "y1932": 0.50357653791130186, "y1931": 0.40973204911672068, "y1930": 0.5716911764705882, "y1929": 0.62092786996274973, "id": 32, "y1939": 0.62901039313149576, "y1938": 0.58882895423699322}, {"neighbors": [9, 38, 40, 44], "y1937": 0.62593576430813813, "y1936": 0.61239743975256666, "y1935": 0.62989685729504619, "y1934": 0.66938595524197997, "y1933": 0.63422690890611111, "y1932": 0.53505007153075823, "y1931": 0.5433879581868809, "y1930": 0.53676470588235292, "y1929": 0.53965458855401283, "id": 31, "y1939": 0.62112311547467447, "y1938": 0.61597355141813115}, {"neighbors": [9, 31], "y1937": 0.52740883844482012, "y1936": 0.5319816143305125, "y1935": 0.53227446612754825, "y1934": 0.55826259508323228, "y1933": 0.53360437047389153, "y1932": 0.45493562231759654, "y1931": 0.44917149769480075, "y1930": 0.44669117647058826, "y1929": 0.44050118523535386, "id": 38, "y1939": 0.54422215832066712, "y1938": 0.52201148425265353}, {"neighbors": [6, 19], "y1937": 1.4122192707075585, "y1936": 1.466042355771296, "y1935": 1.4992010072151469, "y1934": 1.5874765736963952, "y1933": 1.7044848176851735, "y1932": 1.6452074391988556, "y1931": 1.5578582188341628, "y1930": 1.4485294117647058, "y1929": 1.4206569590247204, "id": 37, "y1939": 1.4197099782278273, "y1938": 1.4031668696711328}, {"neighbors": [7, 18, 28, 30, 33, 46], "y1937": 1.2286887225307896, "y1936": 1.2392284891962713, "y1935": 1.2016851484189628, "y1934": 1.2752728475361041, "y1933": 1.2715029856435016, "y1932": 1.2846924177396279, "y1931": 1.3146482859360022, "y1930": 1.3088235294117647, "y1929": 1.2548594649508973, "id": 36, "y1939": 1.1850634679373948, "y1938": 1.1755698625369757}, {"neighbors": [4, 10, 26, 45], "y1937": 1.0741366819608791, "y1936": 1.1299454443919412, "y1935": 1.0645489322550965, "y1934": 1.1615036930878624, "y1933": 1.091602083598018, "y1932": 1.0844062947067239, "y1931": 1.1064956406628019, "y1930": 1.1158088235294117, "y1929": 1.0858110396207246, "id": 35, "y1939": 1.1259088855112354, "y1938": 1.1087523925526361}, {"neighbors": [3, 5, 14, 23, 29, 41], "y1937": 0.72639459067858003, "y1936": 0.66188410155075383, "y1935": 0.69265410875986633, "y1934": 0.66674016095248601, "y1933": 0.67691525854402235, "y1932": 0.61802575107296143, "y1931": 0.65951522344456115, "y1930": 0.67647058823529416, "y1929": 0.73958686081950553, "id": 34, "y1939": 0.68816497555765521, "y1938": 0.72246389420567247}, {"neighbors": [12, 15, 20, 36, 46], "y1937": 1.2518715286162763, "y1936": 1.2227329352635421, "y1935": 1.1993608057721175, "y1934": 1.2038364017197662, "y1933": 1.1739296150425613, "y1932": 1.1444921316165952, "y1931": 1.2335783083032821, "y1930": 1.2150735294117647, "y1929": 1.2532339993227226, "id": 33, "y1939": 1.2126689397362691, "y1938": 1.1713937706629545}] \ No newline at end of file From c7e4baa4aa8c50ba2f86116c1f9581b39af6805d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 20 May 2016 11:47:12 +0200 Subject: [PATCH 048/183] Fix instructions to update/install the extension --- CONTRIBUTING.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcdde4a..a8dc2db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,8 +45,8 @@ source envs/dev/bin/activate Update extension in a working database with: -* `ALTER EXTENSION crankshaft VERSION TO 'current';` - `ALTER EXTENSION crankshaft VERSION TO 'dev';` +* `ALTER EXTENSION crankshaft UPDATE TO 'current';` + `ALTER EXTENSION crankshaft UPDATE TO 'dev';` Note: we keep the current development version install as 'dev' always; we update through the 'current' alias to allow changing the extension @@ -58,7 +58,10 @@ should be dropped manually before the update. If the extension has not previously been installed in a database, it can be installed directly with: -* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` +* `CREATE EXTENSION IF NOT EXISTS plpythonu;` + `CREATE EXTENSION IF NOT EXISTS postgis;` + `CREATE EXTENSION IF NOT EXISTS cartodb;` + `CREATE EXTENSION crankshaft WITH VERSION 'dev';` Note: the development extension uses the development python virtual environment automatically. From 7a1eb6b9b69e84782d8f905f5fe98e2c47881258 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 25 May 2016 11:03:58 -0400 Subject: [PATCH 049/183] adding debug to mock plpy --- src/py/crankshaft/test/mock_plpy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py index 63c88f6..3a3aaea 100644 --- a/src/py/crankshaft/test/mock_plpy.py +++ b/src/py/crankshaft/test/mock_plpy.py @@ -24,6 +24,9 @@ class MockPlPy: def notice(self, msg): self.notices.append(msg) + def debug(self, msg): + self.notices.append(msg) + def info(self, msg): self.infos.append(msg) From fd7aa4140a9ba85232d853bb4653d088188c2280 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 25 May 2016 12:01:55 -0400 Subject: [PATCH 050/183] change function name to match requirements --- src/pg/sql/11_markov.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 09cca98..e804d6c 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -6,21 +6,21 @@ -- 2 | Pt2 | 11.0 | 13.2 | 12.5 -- ... -- Sample Function call: --- SELECT cdb_spatial_markov('SELECT * FROM real_estate', --- Array['date_1', 'date_2', 'date_3']) +-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) CREATE OR REPLACE FUNCTION - cdb_spatial_markov ( + CDB_SpatialMarkov ( subquery TEXT, - time_cols text[], - num_time_per_bin int DEFAULT 1, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs int DEFAULT 5) -RETURNS TABLE (trend numeric, trend_up numeric, trend_down numeric, volatility numeric, ids int) + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, ids INT) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') @@ -28,7 +28,7 @@ AS $$ ## TODO: use named parameters or a dictionary - return spatial_markov_trend(subquery, time_cols, num_time_per_bin, permutations, geom_col, id_col, w_type, num_ngbrs) + return spatial_markov_trend(subquery, time_cols, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format From e80fdca7fcfd605cef19f209d6f4816578246f47 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 25 May 2016 12:02:47 -0400 Subject: [PATCH 051/183] removing time-binning options, reorganizes signature --- .../crankshaft/space_time_dynamics/markov.py | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 1600583..3e39d49 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -8,23 +8,22 @@ import pysal as ps import plpy import crankshaft.pysal_utils as pu -def spatial_markov_trend(subquery, time_cols, num_time_per_bin, - permutations, geom_col, id_col, w_type, num_ngbrs): +def spatial_markov_trend(subquery, time_cols, num_classes = 7, + w_type = 'knn', num_ngbrs = 5, permutations = 999, + geom_col = 'the_geom', id_col = 'cartodb_id'): """ Predict the trends of a unit based on: 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) 2. average class of its neighbors Inputs: - - @param subquery string: e.g., SELECT * FROM table_name - @param time_cols list (string): list of strings of column names - @param num_time_per_bin int: number of bins to divide # of time columns into - @param permutations int: number of permutations for test stats - @param geom_col string: name of column which contains the geometries - @param id_col string: name of column which has the ids of the table - @param w_type string: weight type ('knn' or 'queen') - @param num_ngbrs int: number of neighbors (if knn type) + @param subquery string: e.g., SELECT the_geom, cartodb_id, interesting_time_column FROM table_name + @param time_cols list of strings: list of strings of column names + @param w_type string (optional): weight type ('knn' or 'queen') + @param num_ngbrs int (optional): number of neighbors (if knn type) + @param permutations int (optional): number of permutations for test stats + @param geom_col string (optional): name of column which contains the geometries + @param id_col string (optional): name of column which has the ids of the table Outputs: @param trend_up float: probablity that a geom will move to a higher class @@ -34,8 +33,8 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, @param """ - if num_time_per_bin < 1: - plpy.error('Error: number of time bins must be >= 1') + if len(time_cols) < 2: + plpy.error('More than one time column needs to be passed') qvals = {"id_col": id_col, "time_cols": time_cols, @@ -43,13 +42,15 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, "subquery": subquery, "num_ngbrs": num_ngbrs} - query = pu.construct_neighbor_query(w_type, qvals) - try: - query_result = plpy.execute(query) + query_result = plpy.execute( + pu.construct_neighbor_query(w_type, qvals) + ) + if len(query_result) == 0: + return zip([None], [None], [None], [None], [None]) except plpy.SPIError, err: - plpy.notice('** Query failed with exception %s: %s' % (err, query)) - plpy.error('Spatial Markov failed: check the input parameters') + plpy.debug('Query failed with exception %s: %s' % (err, query)) + plpy.error('Query failed, check the input parameters') return zip([None], [None], [None], [None], [None]) ## build weight @@ -57,34 +58,33 @@ def spatial_markov_trend(subquery, time_cols, num_time_per_bin, ## prep time data t_data = get_time_data(query_result, time_cols) - ## rebin time data - if num_time_per_bin > 1: - ## rebin - t_data = rebin_data(t_data, int(num_time_per_bin)) - print 'shape of t_data %d, %d' % t_data.shape - print 'number of weight objects: %d, %d' % (weights.sparse).shape - print 'first num elements: %f' % t_data[0, 0] + plpy.debug('shape of t_data %d, %d' % t_data.shape) + plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape) + plpy.debug('first num elements: %f' % t_data[0, 0]) # ls = ps.lag_spatial(weights, t_data) sp_markov_result = ps.Spatial_Markov(t_data, weights, - k=7, + k=num_classes, fixed=False, permutations=permutations) ## get lag classes - lag_classes = ps.Quantiles(ps.lag_spatial(weights, t_data[:, -1]), k=7).yb + lag_classes = ps.Quantiles( + ps.lag_spatial(weights, t_data[:, -1]), + k=num_classes).yb ## look up probablity distribution for each unit according to class and lag class - prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1]) + prob_dist = get_prob_dist(sp_markov_result.P, + lag_classes, + sp_markov_result.classes[:, -1]) ## find the ups and down and overall distribution of each cell trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1]) ## output the results - return zip(trend, trend_up, trend_down, volatility, weights.id_order) def get_time_data(markov_data, time_cols): @@ -95,6 +95,7 @@ def get_time_data(markov_data, time_cols): return np.array([[x['attr' + str(i)] for x in markov_data] for i in range(1, num_attrs+1)], dtype=float).transpose() +## not currently used def rebin_data(time_data, num_time_per_bin): """ Convert an n x l matrix into an (n/m) x l matrix where the values are @@ -130,6 +131,7 @@ def rebin_data(time_data, num_time_per_bin): return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) for i in range(n_max)]).T + def get_prob_dist(transition_matrix, lag_indices, unit_indices): """ Given an array of transition matrices, look up the probability @@ -168,7 +170,10 @@ def get_prob_stats(prob_dist, unit_indices): for i in range(num_elements): trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() trend_down[i] = prob_dist[i, :unit_indices[i]].sum() - trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + if prob_dist[i, unit_indices[i]] > 0.0: + trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + else: + trend[i] = None ## calculate volatility of distribution volatility = prob_dist.std(axis=1) From 408e34cd385873f8a373d8172102266b3a751bce Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 10:53:33 -0400 Subject: [PATCH 052/183] newline --- src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py index f45ffdf..f6be2b2 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -1 +1 @@ -from markov import * \ No newline at end of file +from markov import * From 7a7cbcf33fe04b395f34d981e594d344fc70b1f3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 09:49:09 -0400 Subject: [PATCH 053/183] adds test for get_time_data --- .../test/test_space_time_dynamics.py | 219 ++++++++++++++++-- 1 file changed, 201 insertions(+), 18 deletions(-) diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py index 819852b..d702858 100644 --- a/src/py/crankshaft/test/test_space_time_dynamics.py +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -26,7 +26,7 @@ class SpaceTimeTests(unittest.TestCase): "geom_col": "the_geom", "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors_markov.json')).read()) - # self.moran_data = json.loads(open(fixture_file('markov.json')).read()) + self.markov_data = json.loads(open(fixture_file('markov.json')).read()) self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T @@ -58,28 +58,211 @@ class SpaceTimeTests(unittest.TestCase): [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] ) - def test_spatial_markov(self): - """Test Spatial Markov.""" - data = [ { 'id': d['id'], - 'attr1': d['y1929'], - 'attr2': d['y1930'], - 'attr3': d['y1931'], - 'neighbors': d['neighbors'] } for d in self.neighbors_data] + # def test_spatial_markov(self): + # """Test Spatial Markov.""" + # data = [ { 'id': d['id'], + # 'attr1': d['y1995'], + # 'attr2': d['y1996'], + # 'attr3': d['y1997'], + # 'attr4': d['y1998'], + # 'attr5': d['y1999'], + # 'attr6': d['y2000'], + # 'attr7': d['y2001'], + # 'attr8': d['y2002'], + # 'attr9': d['y2003'], + # 'attr10': d['y2004'], + # 'attr11': d['y2005'], + # 'attr12': d['y2006'], + # 'attr13': d['y2007'], + # 'attr14': d['y2008'], + # 'attr15': d['y2009'], + # 'neighbors': d['neighbors'] } for d in self.neighbors_data] + # print(str(data[0])) + # plpy._define_result('select', data) + # random_seeds.set_random_seeds(1234) + # + # result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 7, 'knn', 5, 99, 'the_geom', 'cartodb_id') + # + # print 'result == None? ', result == None + # result = [(row[0], row[1], row[2], row[3], row[4]) for row in result] + # print result[0] + # expected = self.markov_data + # for ([res_trend, res_up, res_down, res_vol, res_id], + # [exp_trend, exp_up, exp_down, exp_vol, exp_id] + # ) in zip(result, expected): + # self.assertAlmostEqual(res_trend, exp_trend) - plpy._define_result('select', data) - random_seeds.set_random_seeds(1234) + def test_get_time_data(self): + """Test get_time_data""" + data = [ { 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'] } for d in self.neighbors_data] - result = std.spatial_markov_trend('subquery', ['y1929', 'y1930', 'y1931', 'y1932', 'y1933', 'y1934', 'y1935', 'y1936', 'y1937', 'y1938', 'y1939'], 1, 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = std.get_time_data(data, ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']) - print 'result == None? ', result == None - result = [(row[0], row[1]) for row in result] - print result[0] - assertTrue(result[0] == None) - # expected = self.moran_data - # for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): - # self.assertAlmostEqual(res_val, exp_val) + ## expected was prepared from PySAL example: + ### f = ps.open(ps.examples.get_path("usjoin.csv")) + ### pci = np.array([f.by_col[str(y)] for y in range(1995, 2010)]).transpose() + ### rpci = pci / (pci.mean(axis = 0)) + expected = np.array([[ 0.87654416, 0.863147, 0.85637567, 0.84811668, 0.8446154, 0.83271652 + , 0.83786314, 0.85012593, 0.85509656, 0.86416612, 0.87119375, 0.86302631 + , 0.86148267, 0.86252252, 0.86746356], + [ 0.9188951, 0.91757931, 0.92333258, 0.92517289, 0.92552388, 0.90746978 + , 0.89830489, 0.89431991, 0.88924794, 0.89815176, 0.91832091, 0.91706054 + , 0.90139505, 0.87897455, 0.86216858], + [ 0.82591007, 0.82548596, 0.81989793, 0.81503235, 0.81731522, 0.78964559 + , 0.80584442, 0.8084998, 0.82258551, 0.82668196, 0.82373724, 0.81814804 + , 0.83675961, 0.83574199, 0.84647177], + [ 1.09088176, 1.08537689, 1.08456418, 1.08415404, 1.09898841, 1.14506948 + , 1.12151133, 1.11160697, 1.10888621, 1.11399806, 1.12168029, 1.13164797 + , 1.12958508, 1.11371818, 1.09936775], + [ 1.10731446, 1.11373944, 1.13283638, 1.14472559, 1.15910025, 1.16898201 + , 1.17212488, 1.14752303, 1.11843284, 1.11024964, 1.11943471, 1.11736468 + , 1.10863242, 1.09642516, 1.07762337], + [ 1.42269757, 1.42118434, 1.44273502, 1.43577571, 1.44400684, 1.44184737 + , 1.44782832, 1.41978227, 1.39092208, 1.4059372, 1.40788646, 1.44052766 + , 1.45241216, 1.43306098, 1.4174431 ], + [ 1.13073885, 1.13110513, 1.11074708, 1.13364636, 1.13088149, 1.10888138 + , 1.11856629, 1.13062931, 1.11944984, 1.12446239, 1.11671008, 1.10880034 + , 1.08401709, 1.06959206, 1.07875225], + [ 1.04706124, 1.04516831, 1.04253372, 1.03239987, 1.02072545, 0.99854316 + , 0.9880258, 0.99669587, 0.99327676, 1.01400905, 1.03176742, 1.040511 + , 1.01749645, 0.9936394, 0.98279746], + [ 0.98996986, 1.00143564, 0.99491, 1.00188408, 1.00455845, 0.99127006 + , 0.97925917, 0.9683482, 0.95335147, 0.93694787, 0.94308213, 0.92232874 + , 0.91284091, 0.89689833, 0.88928858], + [ 0.87418391, 0.86416601, 0.84425695, 0.8404494, 0.83903044, 0.8578708 + , 0.86036185, 0.86107306, 0.8500772, 0.86981998, 0.86837929, 0.87204141 + , 0.86633032, 0.84946077, 0.83287146], + [ 1.14196118, 1.14660262, 1.14892712, 1.14909594, 1.14436624, 1.14450183 + , 1.12349752, 1.12596664, 1.12213996, 1.1119989, 1.10257792, 1.10491258 + , 1.11059842, 1.10509795, 1.10020097], + [ 0.97282463, 0.96700147, 0.96252588, 0.9653878, 0.96057687, 0.95831051 + , 0.94480909, 0.94804195, 0.95430286, 0.94103989, 0.92122519, 0.91010201 + , 0.89280392, 0.89298243, 0.89165385], + [ 0.94325468, 0.96436902, 0.96455242, 0.95243009, 0.94117647, 0.9480927 + , 0.93539182, 0.95388718, 0.94597005, 0.96918424, 0.94781281, 0.93466815 + , 0.94281559, 0.96520315, 0.96715441], + [ 0.97478408, 0.98169225, 0.98712809, 0.98474769, 0.98559897, 0.98687073 + , 0.99237486, 0.98209969, 0.9877653, 0.97399471, 0.96910087, 0.98416665 + , 0.98423613, 0.99823861, 0.99545704], + [ 0.85570269, 0.85575915, 0.85986132, 0.85693406, 0.8538012, 0.86191535 + , 0.84981451, 0.85472102, 0.84564835, 0.83998883, 0.83478547, 0.82803648 + , 0.8198736, 0.82265395, 0.8399404 ], + [ 0.87022047, 0.85996258, 0.85961813, 0.85689572, 0.83947136, 0.82785597 + , 0.86008789, 0.86776298, 0.86720209, 0.8676334, 0.89179317, 0.94202108 + , 0.9422231, 0.93902708, 0.94479184], + [ 0.90134907, 0.90407738, 0.90403991, 0.90201769, 0.90399238, 0.90906632 + , 0.92693339, 0.93695966, 0.94242697, 0.94338265, 0.91981796, 0.91108804 + , 0.90543476, 0.91737138, 0.94793657], + [ 1.1977611, 1.18222564, 1.18439158, 1.18267865, 1.19286723, 1.20172869 + , 1.21328691, 1.22624778, 1.22397075, 1.23857042, 1.24419893, 1.23929384 + , 1.23418676, 1.23626739, 1.26754398], + [ 1.24919678, 1.25754773, 1.26991161, 1.28020651, 1.30625667, 1.34790023 + , 1.34399863, 1.32575181, 1.30795492, 1.30544841, 1.30303302, 1.32107766 + , 1.32936244, 1.33001241, 1.33288462], + [ 1.06768004, 1.03799276, 1.03637303, 1.02768449, 1.03296093, 1.05059016 + , 1.03405057, 1.02747623, 1.03162734, 0.9961416, 0.97356208, 0.94241549 + , 0.92754547, 0.92549227, 0.92138102], + [ 1.09475614, 1.11526796, 1.11654299, 1.13103948, 1.13143264, 1.13889622 + , 1.12442212, 1.13367018, 1.13982256, 1.14029944, 1.11979401, 1.10905389 + , 1.10577769, 1.11166825, 1.09985155], + [ 0.76530058, 0.76612841, 0.76542451, 0.76722683, 0.76014284, 0.74480073 + , 0.76098396, 0.76156903, 0.76651952, 0.76533288, 0.78205934, 0.76842416 + , 0.77487118, 0.77768683, 0.78801192], + [ 0.98391336, 0.98075816, 0.98295341, 0.97386015, 0.96913803, 0.97370819 + , 0.96419154, 0.97209861, 0.97441313, 0.96356162, 0.94745352, 0.93965462 + , 0.93069645, 0.94020973, 0.94358232], + [ 0.83561828, 0.82298088, 0.81738502, 0.81748588, 0.80904801, 0.80071489 + , 0.83358256, 0.83451613, 0.85175032, 0.85954307, 0.86790024, 0.87170334 + , 0.87863799, 0.87497981, 0.87888675], + [ 0.98845573, 1.02092428, 0.99665283, 0.99141823, 0.99386619, 0.98733195 + , 0.99644997, 0.99669587, 1.02559097, 1.01116651, 0.99988024, 0.97906749 + , 0.99323123, 1.00204939, 0.99602148], + [ 1.14930913, 1.15241949, 1.14300962, 1.14265542, 1.13984683, 1.08312397 + , 1.05192626, 1.04230892, 1.05577278, 1.08569751, 1.12443486, 1.08891079 + , 1.08603695, 1.05997314, 1.02160943], + [ 1.11368269, 1.1057147, 1.11893431, 1.13778669, 1.1432272, 1.18257029 + , 1.16226243, 1.16009196, 1.14467789, 1.14820235, 1.12386598, 1.12680236 + , 1.12357937, 1.1159258, 1.12570828], + [ 1.30379431, 1.30752186, 1.31206366, 1.31532267, 1.30625667, 1.31210239 + , 1.29989156, 1.29203193, 1.27183516, 1.26830786, 1.2617743, 1.28656675 + , 1.29734097, 1.29390205, 1.29345446], + [ 0.83953719, 0.82701448, 0.82006005, 0.81188876, 0.80294864, 0.78772975 + , 0.82848011, 0.8259679, 0.82435705, 0.83108634, 0.84373784, 0.83891093 + , 0.84349247, 0.85637272, 0.86539395], + [ 1.23450087, 1.2426022, 1.23537935, 1.23581293, 1.24522626, 1.2256767 + , 1.21126648, 1.19377804, 1.18355337, 1.19674434, 1.21536573, 1.23653297 + , 1.27962009, 1.27968392, 1.25907738], + [ 0.9769662, 0.97400719, 0.98035944, 0.97581531, 0.95543282, 0.96480308 + , 0.94686376, 0.93679073, 0.92540049, 0.92988835, 0.93442917, 0.92100464 + , 0.91475304, 0.90249622, 0.9021363 ], + [ 0.84986886, 0.8986851, 0.84295997, 0.87280534, 0.85659368, 0.88937573 + , 0.894401, 0.90448993, 0.95495898, 0.92698333, 0.94745352, 0.92562488 + , 0.96635366, 1.02520312, 1.0394296 ], + [ 1.01922808, 1.00258203, 1.00974428, 1.00303417, 0.99765073, 1.00759019 + , 0.99192968, 0.99747298, 0.99550759, 0.97583768, 0.9610168, 0.94779638 + , 0.93759089, 0.93353431, 0.94121705], + [ 0.86367411, 0.85558932, 0.85544346, 0.85103025, 0.84336613, 0.83434854 + , 0.85813595, 0.84667961, 0.84374558, 0.85951183, 0.87194227, 0.89455097 + , 0.88283929, 0.90349491, 0.90600675], + [ 1.00947534, 1.00411055, 1.00698819, 0.99513687, 0.99291086, 1.00581626 + , 0.98850522, 0.99291168, 0.98983209, 0.97511924, 0.96134615, 0.96382634 + , 0.95011401, 0.9434686, 0.94637765], + [ 1.05712571, 1.05459419, 1.05753012, 1.04880786, 1.05103857, 1.04800023 + , 1.03024941, 1.04200483, 1.0402554, 1.03296979, 1.02191682, 1.02476275 + , 1.02347523, 1.02517684, 1.04359571], + [ 1.07084189, 1.06669497, 1.07937623, 1.07387988, 1.0794043, 1.0531801 + , 1.07452771, 1.09383478, 1.1052447, 1.10322136, 1.09167939, 1.08772756 + , 1.08859544, 1.09177338, 1.1096083 ], + [ 0.86719222, 0.86628896, 0.86675156, 0.86425632, 0.86511809, 0.86287327 + , 0.85169796, 0.85411285, 0.84886336, 0.84517414, 0.84843858, 0.84488343 + , 0.83374329, 0.82812044, 0.82878599], + [ 0.88389211, 0.92288667, 0.90282398, 0.91229186, 0.92023286, 0.92652175 + , 0.94278865, 0.93682452, 0.98655146, 0.992237, 0.9798497, 0.93869677 + , 0.96947771, 1.00362626, 0.98102351], + [ 0.97082064, 0.95320233, 0.94534081, 0.94215593, 0.93967, 0.93092109 + , 0.92662519, 0.93412152, 0.93501274, 0.92879506, 0.92110542, 0.91035556 + , 0.90430364, 0.89994694, 0.90073864], + [ 0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824, 0.98882205 + , 0.97662234, 0.95601578, 0.94905385, 0.94934888, 0.97152609, 0.97163004 + , 0.9700702, 0.97158948, 0.95884908], + [ 0.83980439, 0.84726737, 0.85747, 0.85467221, 0.8556751, 0.84818516 + , 0.85265681, 0.84502402, 0.82645665, 0.81743586, 0.83550406, 0.83338919 + , 0.83511679, 0.82136617, 0.80921874], + [ 0.95118156, 0.9466212, 0.94688098, 0.9508583, 0.9512441, 0.95440787 + , 0.96364363, 0.96804412, 0.97136214, 0.97583768, 0.95571724, 0.96895368 + , 0.97001634, 0.97082733, 0.98782366], + [ 1.08910044, 1.08248968, 1.08492895, 1.08656923, 1.09454249, 1.10558188 + , 1.1214086, 1.12292577, 1.13021031, 1.13342735, 1.14686068, 1.14502975 + , 1.14474747, 1.14084037, 1.16142926], + [ 1.06336033, 1.07365823, 1.08691496, 1.09764846, 1.11669863, 1.11856702 + , 1.09764283, 1.08815849, 1.08044313, 1.09278827, 1.07003204, 1.08398066 + , 1.09831768, 1.09298232, 1.09176125], + [ 0.79772065, 0.78829196, 0.78581151, 0.77615922, 0.77035744, 0.77751194 + , 0.79902974, 0.81437881, 0.80788828, 0.79603865, 0.78966436, 0.79949807 + , 0.80172182, 0.82168155, 0.85587911], + [ 1.0052447, 1.00007696, 1.00475899, 1.00613942, 1.00639561, 1.00162979 + , 0.99860739, 1.00814981, 1.00574316, 0.99030032, 0.97682565, 0.97292596 + , 0.96519561, 0.96173403, 0.95890284], + [ 0.95808419, 0.9382568, 0.9654441, 0.95561201, 0.96987289, 0.96608031 + , 0.99727185, 1.00781194, 1.03484236, 1.05333619, 1.0983263, 1.1704974 + , 1.17025154, 1.18730553, 1.14242645]]) + self.assertTrue(np.allclose(result, expected)) def test_rebin_data(self): """Test rebin_data""" From 0eb3db3c1dbccdf2387af4ecb2aa5688d891cb49 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 12:12:52 -0400 Subject: [PATCH 054/183] adding fixtures --- src/py/crankshaft/test/fixtures/markov.json | 1 + src/py/crankshaft/test/fixtures/neighbors_markov.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 src/py/crankshaft/test/fixtures/markov.json diff --git a/src/py/crankshaft/test/fixtures/markov.json b/src/py/crankshaft/test/fixtures/markov.json new file mode 100644 index 0000000..d60e4e0 --- /dev/null +++ b/src/py/crankshaft/test/fixtures/markov.json @@ -0,0 +1 @@ +[[0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 0], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 1], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 2], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 3], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 4], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 5], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 6], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 7], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 8], [0.19047619047619049, 0.16, 0.0, 0.32594478059941379, 9], [-0.23529411764705882, 0.0, 0.19047619047619047, 0.31356338348865387, 10], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 11], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 12], [0.027777777777777783, 0.11111111111111112, 0.088888888888888892, 0.30339641183779581, 13], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 14], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 15], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 16], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 17], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 18], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 19], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 20], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 21], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 22], [-0.16666666666666663, 0.18181818181818182, 0.27272727272727271, 0.20246415864836445, 23], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 24], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 25], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 26], [-0.043478260869565216, 0.0, 0.041666666666666664, 0.37950991789118999, 27], [0.22222222222222221, 0.18181818181818182, 0.0, 0.31701083225750354, 28], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 29], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 30], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 31], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 32], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 33], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 34], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 35], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 36], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 37], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 38], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 39], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 40], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 41], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 42], [0.0, 0.0, 0.0, 0.40000000000000002, 43], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 44], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 45], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 46], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 47]] diff --git a/src/py/crankshaft/test/fixtures/neighbors_markov.json b/src/py/crankshaft/test/fixtures/neighbors_markov.json index d83122a..45a20e7 100644 --- a/src/py/crankshaft/test/fixtures/neighbors_markov.json +++ b/src/py/crankshaft/test/fixtures/neighbors_markov.json @@ -1 +1 @@ -[{"neighbors": [19, 27, 30], "y1937": 0.93697174595508337, "y1936": 0.9711757377894239, "y1935": 0.96227785579390823, "y1934": 1.0133392128761989, "y1933": 1.0306187269724305, "y1932": 1.044349070100143, "y1931": 1.0385721458894417, "y1930": 1.0588235294117647, "y1929": 1.0305452082627835, "id": 43, "y1939": 0.96816333237481, "y1938": 0.95423699321385069}, {"neighbors": [15, 18, 31, 40, 46], "y1937": 0.81719391451340251, "y1936": 0.8041582542205421, "y1935": 0.81351992639581616, "y1934": 0.84665417263807741, "y1933": 0.86901283191462331, "y1932": 0.81258941344778257, "y1931": 0.81069977632720136, "y1930": 0.70588235294117652, "y1929": 0.7054520826278361, "id": 44, "y1939": 0.83999507045146449, "y1938": 0.81433791543413947}, {"neighbors": [5, 13, 14, 23, 39, 48], "y1937": 0.80173871045641154, "y1936": 0.81652991967008892, "y1935": 0.95065614255968234, "y1934": 0.68526072097894397, "y1933": 0.83852115360182955, "y1932": 0.87839771101573672, "y1931": 0.90491623681928157, "y1930": 0.95772058823529416, "y1929": 0.96877751439214355, "id": 25, "y1939": 0.78872776568212632, "y1938": 0.84565860448929875}, {"neighbors": [2, 4, 10, 35, 42], "y1937": 1.4721081864283989, "y1936": 1.7382189956613256, "y1935": 1.5294174616241343, "y1934": 1.4446036820637196, "y1933": 1.5093380764832931, "y1932": 1.5736766809728182, "y1931": 1.4285844707171225, "y1930": 1.53125, "y1929": 1.4109041652556722, "id": 26, "y1939": 1.6977365156307769, "y1938": 1.6286758308682789}, {"neighbors": [17, 19, 43], "y1937": 1.091523786524994, "y1936": 1.1072640577344388, "y1935": 1.1575226381289041, "y1934": 1.2593980817991401, "y1933": 1.2684538178122222, "y1932": 1.2217453505007152, "y1931": 1.2226229059204821, "y1930": 1.1893382352941178, "y1929": 1.11506942092787, "id": 27, "y1939": 1.1061906913691821, "y1938": 1.1129284844266574}, {"neighbors": [7, 30, 36], "y1937": 1.4431296788215406, "y1936": 1.4619184672881136, "y1935": 1.4527141542782431, "y1934": 1.5160401278800575, "y1933": 1.5947147757591158, "y1932": 1.6795422031473533, "y1931": 1.6126352307481628, "y1930": 1.556985294117647, "y1929": 1.4921774466644089, "id": 28, "y1939": 1.4768927412397814, "y1938": 1.4553680180963982}, {"neighbors": [13, 32, 39, 47], "y1937": 1.0432262738468969, "y1936": 0.97323768203101502, "y1935": 1.0482785337271803, "y1934": 0.94719435563884913, "y1933": 0.93914369203404913, "y1932": 1.03862660944206, "y1931": 1.0013237777879218, "y1930": 1.0147058823529411, "y1929": 0.97365391127666778, "id": 21, "y1939": 1.0194306371441482, "y1938": 1.0314946928832434}, {"neighbors": [1, 3, 16, 40], "y1937": 0.43274571359574981, "y1936": 0.47218523132436957, "y1935": 0.41140864849159847, "y1934": 0.46036820637195458, "y1933": 0.3994409858975988, "y1932": 0.36337625178826893, "y1931": 0.38343908339800065, "y1930": 0.37132352941176472, "y1929": 0.46488316965797494, "id": 22, "y1939": 0.40422297991208972, "y1938": 0.41969723333913345}, {"neighbors": [3, 11, 13, 14, 15, 25, 34, 40], "y1937": 0.98140545761893261, "y1936": 0.9608660165814682, "y1935": 0.97622391167497935, "y1934": 0.97100650424429502, "y1933": 1.0184220556473129, "y1932": 1.044349070100143, "y1931": 1.0758205139909618, "y1930": 1.03125, "y1929": 1.0094141550965119, "id": 23, "y1939": 0.9937969847594792, "y1938": 0.99182182008004172}, {"neighbors": [10, 32, 39, 48], "y1937": 0.98913305964742815, "y1936": 0.97942351475578848, "y1935": 1.1063870998983099, "y1934": 0.96306912137581313, "y1933": 0.90865201372125526, "y1932": 0.96995708154506433, "y1931": 0.83699274204592145, "y1930": 0.92095588235294112, "y1929": 0.96227565187944453, "id": 24, "y1939": 1.0509797477714333, "y1938": 1.0795197494344875}, {"neighbors": [11, 13, 20, 21], "y1937": 1.0644771794252597, "y1936": 1.0680871171442072, "y1935": 1.0715219601956321, "y1934": 1.005401830007717, "y1933": 1.0153728878160335, "y1932": 1.0357653791130186, "y1931": 1.0276167435066417, "y1930": 1.0808823529411764, "y1929": 1.0939383677615984, "id": 47, "y1939": 1.0115433594873271, "y1938": 1.0586392900643813}, {"neighbors": [5, 10, 24, 25, 39, 42], "y1937": 1.172663607824197, "y1936": 1.1361312771167147, "y1935": 1.1528739528352137, "y1934": 1.0874214529820307, "y1933": 1.13124126540465, "y1932": 1.0701001430615165, "y1931": 1.0429543068425617, "y1930": 1.0753676470588236, "y1929": 1.0971892990179477, "id": 48, "y1939": 1.1574579961385203, "y1938": 1.1713937706629545}, {"neighbors": [10, 35], "y1937": 1.1572084037672059, "y1936": 1.173246273465355, "y1935": 1.1389278969541425, "y1934": 1.1720868702458385, "y1933": 1.146487104561047, "y1932": 1.150214592274678, "y1931": 1.1700369744830419, "y1930": 1.2095588235294117, "y1929": 1.2044700304774805, "id": 45, "y1939": 1.2106971203220638, "y1938": 1.2152427353401773}, {"neighbors": [15, 18, 33, 36, 44], "y1937": 0.8075344119777832, "y1936": 0.8041582542205421, "y1935": 0.7833034719868287, "y1934": 0.82813361261161944, "y1933": 0.78973446830135952, "y1932": 0.73533619456366239, "y1931": 0.78002464965536134, "y1930": 0.75, "y1929": 0.74771418896037922, "id": 46, "y1939": 0.76506593271166257, "y1938": 0.77257699669392721}, {"neighbors": [2, 5, 34, 41, 42], "y1937": 0.69934798357884564, "y1936": 0.70724687486575877, "y1935": 0.67870805287879521, "y1934": 0.65351118950501597, "y1933": 0.64337441239994919, "y1932": 0.59513590844062947, "y1931": 0.63322225772584106, "y1930": 0.61397058823529416, "y1929": 0.66644090755164231, "id": 29, "y1939": 0.70393953087129768, "y1938": 0.70575952670958764}, {"neighbors": [6, 19, 28, 36, 43], "y1937": 1.6189326249698142, "y1936": 1.666050947205636, "y1935": 1.6781753910222263, "y1934": 1.7991401168559145, "y1933": 1.908779062380892, "y1932": 1.9341917024320459, "y1931": 1.9303418998493633, "y1930": 1.9025735294117647, "y1929": 1.8725364036572976, "id": 30, "y1939": 1.6267510167193855, "y1938": 1.6474682443013746}, {"neighbors": [3, 16, 29, 34], "y1937": 0.8075344119777832, "y1936": 0.76704325787190164, "y1935": 0.75773570287153158, "y1934": 0.77786352111123369, "y1933": 0.78363613263880072, "y1932": 0.76108726752503575, "y1931": 0.76249600584288135, "y1930": 0.75735294117647056, "y1929": 0.7785980358956992, "id": 41, "y1939": 0.8222486957236167, "y1938": 0.84357055855228813}, {"neighbors": [2, 5, 10, 26, 29, 48], "y1937": 0.85776382516300409, "y1936": 0.95468018385669484, "y1935": 0.90416928962277854, "y1934": 0.82019622974313755, "y1933": 0.90865201372125526, "y1932": 0.87267525035765381, "y1931": 0.80850869585064133, "y1930": 0.9154411764705882, "y1929": 0.89563156112428033, "id": 42, "y1939": 0.90309329170603458, "y1938": 0.92709239603271265}, {"neighbors": [4, 5, 26, 29, 42], "y1937": 0.97367785559043707, "y1936": 0.95261823961510372, "y1935": 0.96692654108759857, "y1934": 0.95777753279682509, "y1933": 0.93914369203404913, "y1932": 0.91845493562231761, "y1931": 0.93997352444424165, "y1930": 0.95588235294117652, "y1929": 0.97527937690484245, "id": 2, "y1939": 0.96619151296060468, "y1938": 0.99808595789107357}, {"neighbors": [8, 9, 22, 40], "y1937": 0.51581743540207681, "y1936": 0.51754800463937456, "y1935": 0.50438235436540602, "y1934": 0.55826259508323228, "y1933": 0.50616185999237706, "y1932": 0.46351931330472101, "y1931": 0.49080202674944085, "y1930": 0.49080882352941174, "y1929": 0.52502539790044023, "id": 1, "y1939": 0.4968984923797396, "y1938": 0.50948320863058982}, {"neighbors": [2, 26, 35], "y1937": 1.5358609031634871, "y1936": 1.589759010266764, "y1935": 1.5340661469178247, "y1934": 1.5954139565648771, "y1933": 1.6648456358785415, "y1932": 1.659513590844063, "y1931": 1.6411192769434428, "y1930": 1.630514705882353, "y1929": 1.6108364375211648, "id": 4, "y1939": 1.5399909624943515, "y1938": 1.6098834174351835}, {"neighbors": [16, 22, 23, 34, 40, 41], "y1937": 0.49456652982371407, "y1936": 0.50930022767300998, "y1935": 0.48113892789695412, "y1934": 0.49476353213537649, "y1933": 0.4787193495108627, "y1932": 0.44921316165951358, "y1931": 0.47108230246040078, "y1930": 0.41911764705882354, "y1929": 0.50389434473416861, "id": 3, "y1939": 0.49098303413712363, "y1938": 0.48233861144945189}, {"neighbors": [19, 30, 37], "y1937": 1.6614344361265394, "y1936": 1.6619270587224537, "y1935": 1.6409859086727034, "y1934": 1.7277036710395768, "y1933": 1.7776648456358786, "y1932": 1.7739628040057225, "y1931": 1.7550554617245631, "y1930": 1.693014705882353, "y1929": 1.6644768032509312, "id": 6, "y1939": 1.648441030275644, "y1938": 1.6057073255611622}, {"neighbors": [2, 14, 25, 29, 34, 42, 48], "y1937": 1.0277710697899058, "y1936": 1.1175737789423943, "y1935": 1.0320081351992638, "y1934": 0.9736522985337891, "y1933": 1.0763562444416213, "y1932": 1.0128755364806867, "y1931": 1.0319989044597617, "y1930": 1.0625, "y1929": 1.0305452082627835, "id": 5, "y1939": 1.0174588177299428, "y1938": 1.0565512441273708}, {"neighbors": [1, 9], "y1937": 0.94083554696933103, "y1936": 0.92787490871601008, "y1935": 0.87395283521379108, "y1934": 0.92073641274390916, "y1933": 0.8781603354084615, "y1932": 0.9127324749642346, "y1931": 0.87205002967088152, "y1930": 0.86397058823529416, "y1929": 0.84199119539451395, "id": 8, "y1939": 0.97605061003163129, "y1938": 0.96050113102488255}, {"neighbors": [18, 28, 36], "y1937": 1.8333735812605652, "y1936": 1.7897676017011039, "y1935": 1.6293641954384774, "y1934": 1.7065373167236249, "y1933": 1.7197306568415704, "y1932": 1.6881258941344779, "y1931": 1.6980873693340028, "y1930": 1.5753676470588236, "y1929": 1.677480528276329, "id": 7, "y1939": 1.7726656533705789, "y1938": 1.6599965199234383}, {"neighbors": [24, 26, 35, 42, 45, 48], "y1937": 0.81719391451340251, "y1936": 0.97942351475578848, "y1935": 0.92741271609123044, "y1934": 1.0662550986660788, "y1933": 0.69216109770041934, "y1932": 0.78397711015736771, "y1931": 0.81946409823344135, "y1930": 0.92463235294117652, "y1929": 0.82411107348459189, "id": 10, "y1939": 0.86168508400772303, "y1938": 0.88950756916652163}, {"neighbors": [1, 8, 31, 38, 40], "y1937": 0.6046848587297754, "y1936": 0.62270716096052237, "y1935": 0.62292382935451063, "y1934": 0.64557380663653408, "y1933": 0.62203023758099352, "y1932": 0.57224606580829762, "y1931": 0.560916601999361, "y1930": 0.56433823529411764, "y1929": 0.5640365729766339, "id": 9, "y1939": 0.60929219898944253, "y1938": 0.60553332173307806}, {"neighbors": [1, 3, 9, 15, 22, 23, 31, 44], "y1937": 0.64525476937937698, "y1936": 0.6268310494437046, "y1935": 0.61362645876712985, "y1934": 0.64821960092602804, "y1933": 0.62203023758099352, "y1932": 0.5665236051502146, "y1931": 0.60692929200712109, "y1930": 0.59742647058823528, "y1929": 0.61442600745005072, "id": 40, "y1939": 0.61323583781785318, "y1938": 0.62641378110318424}, {"neighbors": [13, 21, 24, 25, 32, 48], "y1937": 0.62400386380101425, "y1936": 0.50311439494823662, "y1935": 0.71822187787516345, "y1934": 0.48682614926689455, "y1933": 0.39334265023504006, "y1932": 0.54077253218884125, "y1931": 0.52805039485096095, "y1930": 0.67279411764705888, "y1929": 0.69244835760243817, "id": 39, "y1939": 0.68027769790083392, "y1938": 0.6681746998433965}, {"neighbors": [11, 15, 20, 33], "y1937": 1.0567495773967641, "y1936": 0.99179518020533519, "y1935": 0.97854825432182457, "y1934": 0.94984014992834309, "y1933": 0.89645534239613778, "y1932": 0.88698140200286124, "y1931": 0.95969324873328166, "y1930": 0.94485294117647056, "y1929": 0.9866576363020656, "id": 12, "y1939": 1.0214024565583535, "y1938": 0.98555768226900986}, {"neighbors": [12, 13, 15, 23, 47], "y1937": 1.4122192707075585, "y1936": 1.3402637570342368, "y1935": 1.3318483366422933, "y1934": 1.336126116194466, "y1933": 1.3324863422690891, "y1932": 1.390557939914163, "y1931": 1.4702149997717626, "y1930": 1.4834558823529411, "y1929": 1.540941415509651, "id": 11, "y1939": 1.3881608676005424, "y1938": 1.353053767182878}, {"neighbors": [5, 23, 25, 34], "y1937": 0.82685341704902193, "y1936": 0.79797242149576864, "y1935": 0.8414120381579584, "y1934": 0.75934296108477572, "y1933": 0.76229195781984505, "y1932": 0.76108726752503575, "y1931": 0.87862327110056149, "y1930": 0.85845588235294112, "y1929": 0.86474771418896035, "id": 14, "y1939": 0.75323501622643063, "y1938": 0.79972159387506525}, {"neighbors": [11, 21, 23, 25, 39, 47], "y1937": 1.0103839652257909, "y1936": 0.81034408694531546, "y1935": 0.98784562490920536, "y1934": 0.71171866387388383, "y1933": 0.77143946131368324, "y1932": 0.84978540772532185, "y1931": 0.87643219062400146, "y1930": 0.9375, "y1929": 0.94439552996952247, "id": 13, "y1939": 0.93661422174752496, "y1938": 0.95632503915086131}, {"neighbors": [3, 22, 41], "y1937": 0.68196087901473079, "y1936": 0.68044159972507412, "y1935": 0.67405936758510476, "y1934": 0.70113548671590786, "y1933": 0.69216109770041934, "y1932": 0.68955650929899859, "y1931": 0.69676359154608114, "y1930": 0.65257352941176472, "y1929": 0.67294277006434133, "id": 16, "y1939": 0.70985498911391365, "y1938": 0.72663998607969371}, {"neighbors": [11, 12, 23, 33, 40, 44, 46], "y1937": 0.65877807292924417, "y1936": 0.60621160702779331, "y1935": 0.61595080141397507, "y1934": 0.61647006945210014, "y1933": 0.62507940541227291, "y1932": 0.60371959942775388, "y1931": 0.63760441867896112, "y1930": 0.59742647058823528, "y1929": 0.63880799187267179, "id": 15, "y1939": 0.60140492133262136, "y1938": 0.62014964329215239}, {"neighbors": [7, 36, 44, 46], "y1937": 1.2847138372373823, "y1936": 1.2742815413033206, "y1935": 1.2737397704711635, "y1934": 1.3837504134053578, "y1933": 1.4209122093761912, "y1932": 1.4649499284692418, "y1931": 1.3979093440452823, "y1930": 1.3088235294117647, "y1929": 1.2483576024381984, "id": 18, "y1939": 1.3073162716181244, "y1938": 1.3217330781277188}, {"neighbors": [27], "y1937": 0.98526925863318038, "y1936": 1.0433437862451136, "y1935": 0.99946733814343125, "y1934": 1.1006504244295006, "y1933": 1.13124126540465, "y1932": 1.0786838340486409, "y1931": 1.0758205139909618, "y1930": 1.0588235294117647, "y1929": 0.97690484253301724, "id": 17, "y1939": 0.97605061003163129, "y1938": 0.98346963633199924}, {"neighbors": [12, 33, 47], "y1937": 1.32335184737986, "y1936": 1.2763434855449116, "y1935": 1.2319016028279501, "y1934": 1.1985448131407783, "y1933": 1.0580612374539449, "y1932": 1.1273247496423462, "y1931": 1.183183457342402, "y1930": 1.2077205882352942, "y1929": 1.2841178462580425, "id": 20, "y1939": 1.2323871338783223, "y1938": 1.1943622759700714}, {"neighbors": [6, 27, 30, 37, 43], "y1937": 1.4141511712146824, "y1936": 1.4722281884960693, "y1935": 1.4945523219214565, "y1934": 1.6112887223018411, "y1933": 1.7044848176851735, "y1932": 1.753934191702432, "y1931": 1.6630300817090429, "y1930": 1.536764705882353, "y1929": 1.4726718591263122, "id": 19, "y1939": 1.4275972558846486, "y1938": 1.4031668696711328}, {"neighbors": [21, 24, 39], "y1937": 0.6297995653223859, "y1936": 0.48249495253232527, "y1935": 0.63222119994189141, "y1934": 0.47624297210891858, "y1933": 0.44517850336678949, "y1932": 0.50357653791130186, "y1931": 0.40973204911672068, "y1930": 0.5716911764705882, "y1929": 0.62092786996274973, "id": 32, "y1939": 0.62901039313149576, "y1938": 0.58882895423699322}, {"neighbors": [9, 38, 40, 44], "y1937": 0.62593576430813813, "y1936": 0.61239743975256666, "y1935": 0.62989685729504619, "y1934": 0.66938595524197997, "y1933": 0.63422690890611111, "y1932": 0.53505007153075823, "y1931": 0.5433879581868809, "y1930": 0.53676470588235292, "y1929": 0.53965458855401283, "id": 31, "y1939": 0.62112311547467447, "y1938": 0.61597355141813115}, {"neighbors": [9, 31], "y1937": 0.52740883844482012, "y1936": 0.5319816143305125, "y1935": 0.53227446612754825, "y1934": 0.55826259508323228, "y1933": 0.53360437047389153, "y1932": 0.45493562231759654, "y1931": 0.44917149769480075, "y1930": 0.44669117647058826, "y1929": 0.44050118523535386, "id": 38, "y1939": 0.54422215832066712, "y1938": 0.52201148425265353}, {"neighbors": [6, 19], "y1937": 1.4122192707075585, "y1936": 1.466042355771296, "y1935": 1.4992010072151469, "y1934": 1.5874765736963952, "y1933": 1.7044848176851735, "y1932": 1.6452074391988556, "y1931": 1.5578582188341628, "y1930": 1.4485294117647058, "y1929": 1.4206569590247204, "id": 37, "y1939": 1.4197099782278273, "y1938": 1.4031668696711328}, {"neighbors": [7, 18, 28, 30, 33, 46], "y1937": 1.2286887225307896, "y1936": 1.2392284891962713, "y1935": 1.2016851484189628, "y1934": 1.2752728475361041, "y1933": 1.2715029856435016, "y1932": 1.2846924177396279, "y1931": 1.3146482859360022, "y1930": 1.3088235294117647, "y1929": 1.2548594649508973, "id": 36, "y1939": 1.1850634679373948, "y1938": 1.1755698625369757}, {"neighbors": [4, 10, 26, 45], "y1937": 1.0741366819608791, "y1936": 1.1299454443919412, "y1935": 1.0645489322550965, "y1934": 1.1615036930878624, "y1933": 1.091602083598018, "y1932": 1.0844062947067239, "y1931": 1.1064956406628019, "y1930": 1.1158088235294117, "y1929": 1.0858110396207246, "id": 35, "y1939": 1.1259088855112354, "y1938": 1.1087523925526361}, {"neighbors": [3, 5, 14, 23, 29, 41], "y1937": 0.72639459067858003, "y1936": 0.66188410155075383, "y1935": 0.69265410875986633, "y1934": 0.66674016095248601, "y1933": 0.67691525854402235, "y1932": 0.61802575107296143, "y1931": 0.65951522344456115, "y1930": 0.67647058823529416, "y1929": 0.73958686081950553, "id": 34, "y1939": 0.68816497555765521, "y1938": 0.72246389420567247}, {"neighbors": [12, 15, 20, 36, 46], "y1937": 1.2518715286162763, "y1936": 1.2227329352635421, "y1935": 1.1993608057721175, "y1934": 1.2038364017197662, "y1933": 1.1739296150425613, "y1932": 1.1444921316165952, "y1931": 1.2335783083032821, "y1930": 1.2150735294117647, "y1929": 1.2532339993227226, "id": 33, "y1939": 1.2126689397362691, "y1938": 1.1713937706629545}] \ No newline at end of file +[{"neighbors": [10, 7, 21, 23, 1], "y1995": 0.87654416055651474, "y1997": 0.85637566664752718, "y1996": 0.8631470006766887, "y1999": 0.84461540228037335, "y1998": 0.84811668329242784, "y2006": 0.86302631339545688, "y2007": 0.86148266513456728, "y2004": 0.86416611731111015, "y2005": 0.87119374831581786, "y2002": 0.85012592862683589, "y2003": 0.8550965633336135, "y2000": 0.83271652434603094, "y2001": 0.83786313566577242, "id": 0, "y2008": 0.86252252380501315, "y2009": 0.86746356478544273}, {"neighbors": [5, 7, 22, 29, 3], "y1995": 0.91889509774542122, "y1997": 0.92333257900976462, "y1996": 0.91757931190043385, "y1999": 0.92552387732371888, "y1998": 0.92517289327379471, "y2006": 0.91706053906277052, "y2007": 0.90139504820726424, "y2004": 0.89815175749309051, "y2005": 0.91832090781161113, "y2002": 0.89431990798552208, "y2003": 0.88924793576523797, "y2000": 0.90746978227271013, "y2001": 0.89830489127332913, "id": 1, "y2008": 0.87897455159080617, "y2009": 0.86216858051752643}, {"neighbors": [11, 8, 13, 18, 17], "y1995": 0.82591007476914713, "y1997": 0.81989792988843901, "y1996": 0.82548595539161707, "y1999": 0.81731522200916285, "y1998": 0.81503235035017918, "y2006": 0.81814804358939286, "y2007": 0.83675961003285626, "y2004": 0.82668195534569056, "y2005": 0.82373723764184559, "y2002": 0.80849979516360859, "y2003": 0.82258550658074148, "y2000": 0.78964559168205917, "y2001": 0.8058444152731008, "id": 2, "y2008": 0.8357419865626442, "y2009": 0.84647177436289112}, {"neighbors": [4, 14, 9, 5, 12], "y1995": 1.0908817638059434, "y1997": 1.0845641754849344, "y1996": 1.0853768890893893, "y1999": 1.098988414417104, "y1998": 1.0841540389418189, "y2006": 1.1316479722785828, "y2007": 1.1295850763954971, "y2004": 1.1139980568106316, "y2005": 1.1216802898290368, "y2002": 1.1116069731657288, "y2003": 1.1088862051501811, "y2000": 1.1450694824791507, "y2001": 1.1215113292620285, "id": 3, "y2008": 1.1137181812756343, "y2009": 1.0993677488645406}, {"neighbors": [14, 3, 9, 31, 12], "y1995": 1.1073144618319228, "y1997": 1.1328363804627946, "y1996": 1.1137394350312471, "y1999": 1.1591002514611153, "y1998": 1.144725587086376, "y2006": 1.1173646811350333, "y2007": 1.1086324218539598, "y2004": 1.1102496406140896, "y2005": 1.11943471361418, "y2002": 1.1475230282561595, "y2003": 1.1184328424005199, "y2000": 1.1689820101690329, "y2001": 1.1721248787169682, "id": 4, "y2008": 1.0964251552643696, "y2009": 1.0776233718455337}, {"neighbors": [29, 1, 22, 7, 4], "y1995": 1.422697571371182, "y1997": 1.4427350196405593, "y1996": 1.4211843379728528, "y1999": 1.4440068434166562, "y1998": 1.4357757095632602, "y2006": 1.4405276647793266, "y2007": 1.4524121586440921, "y2004": 1.4059372049179741, "y2005": 1.4078864636665769, "y2002": 1.4197822680667809, "y2003": 1.3909220829548647, "y2000": 1.4418473669388905, "y2001": 1.4478283203013527, "id": 5, "y2008": 1.4330609762040207, "y2009": 1.4174430982377491}, {"neighbors": [12, 47, 9, 25, 20], "y1995": 1.1307388498039153, "y1997": 1.1107470843142355, "y1996": 1.1311051255854685, "y1999": 1.130881491772973, "y1998": 1.1336463608751246, "y2006": 1.1088003408832796, "y2007": 1.0840170924825394, "y2004": 1.1244623853593112, "y2005": 1.1167100811401538, "y2002": 1.1306293052597198, "y2003": 1.1194498381213465, "y2000": 1.1088813841947593, "y2001": 1.1185662918783175, "id": 6, "y2008": 1.0695920556329086, "y2009": 1.0787522517402164}, {"neighbors": [21, 1, 22, 10, 0], "y1995": 1.0470612357366649, "y1997": 1.0425337165747406, "y1996": 1.0451683097376836, "y1999": 1.0207254480945218, "y1998": 1.0323998680588111, "y2006": 1.0405109962442973, "y2007": 1.0174964540280445, "y2004": 1.0140090547678748, "y2005": 1.0317674181861733, "y2002": 0.99669586934394627, "y2003": 0.99327675611171373, "y2000": 0.99854316295509526, "y2001": 0.98802579761429143, "id": 7, "y2008": 0.9936394033949828, "y2009": 0.98279746069218921}, {"neighbors": [11, 13, 17, 18, 15], "y1995": 0.98996985668705595, "y1997": 0.99491000469481983, "y1996": 1.0014356415938011, "y1999": 1.0045584503565237, "y1998": 1.0018840754492748, "y2006": 0.92232873520447411, "y2007": 0.91284090705064902, "y2004": 0.93694786512729977, "y2005": 0.94308212820743131, "y2002": 0.96834820215592055, "y2003": 0.95335147249088092, "y2000": 0.99127006477048718, "y2001": 0.97925917470464008, "id": 8, "y2008": 0.89689832627117483, "y2009": 0.88928857608264111}, {"neighbors": [12, 6, 4, 3, 14], "y1995": 0.87418390853652306, "y1997": 0.84425695187978567, "y1996": 0.86416601430334228, "y1999": 0.83903043942542854, "y1998": 0.8404493987171674, "y2006": 0.87204140839730271, "y2007": 0.86633032299764789, "y2004": 0.86981997840756087, "y2005": 0.86837929279319737, "y2002": 0.86107306112852877, "y2003": 0.85007719735663123, "y2000": 0.85787080050645603, "y2001": 0.86036185149249467, "id": 9, "y2008": 0.84946077011565357, "y2009": 0.83287145944123797}, {"neighbors": [0, 7, 21, 23, 22], "y1995": 1.1419611801631209, "y1997": 1.1489271154554144, "y1996": 1.146602624490825, "y1999": 1.1443662376135306, "y1998": 1.1490959392942743, "y2006": 1.1049125811637337, "y2007": 1.1105984164317646, "y2004": 1.1119989015058092, "y2005": 1.1025779214946556, "y2002": 1.1259666377127024, "y2003": 1.1221399558345004, "y2000": 1.144501826035474, "y2001": 1.1234975172649961, "id": 10, "y2008": 1.1050979494645479, "y2009": 1.1002009697391872}, {"neighbors": [8, 13, 18, 17, 2], "y1995": 0.97282462974938089, "y1997": 0.96252588061647382, "y1996": 0.96700147279313231, "y1999": 0.96057686787383312, "y1998": 0.96538780087103548, "y2006": 0.91010201260822066, "y2007": 0.89280392121658247, "y2004": 0.94103988614185807, "y2005": 0.9212251863828258, "y2002": 0.94804194711420009, "y2003": 0.9543028555845573, "y2000": 0.95831051250950716, "y2001": 0.94480908623936988, "id": 11, "y2008": 0.89298242828382146, "y2009": 0.89165384824292859}, {"neighbors": [33, 9, 6, 25, 31], "y1995": 0.94325467991401402, "y1997": 0.96455242154753429, "y1996": 0.96436902092427723, "y1999": 0.94117647058823528, "y1998": 0.95243008993884537, "y2006": 0.9346681464882507, "y2007": 0.94281559150403071, "y2004": 0.96918424441756057, "y2005": 0.94781280876672958, "y2002": 0.95388717527096822, "y2003": 0.94597005193649519, "y2000": 0.94809269652332606, "y2001": 0.93539181553564288, "id": 12, "y2008": 0.965203150896216, "y2009": 0.967154410723015}, {"neighbors": [18, 17, 11, 8, 19], "y1995": 0.97478408425654373, "y1997": 0.98712808751954773, "y1996": 0.98169225257738801, "y1999": 0.985598971191053, "y1998": 0.98474769442356791, "y2006": 0.98416665248276058, "y2007": 0.98423613480079708, "y2004": 0.97399471186978948, "y2005": 0.96910087128357136, "y2002": 0.9820996926750224, "y2003": 0.98776529543110569, "y2000": 0.98687072733199255, "y2001": 0.99237486444837619, "id": 13, "y2008": 0.99823861244053191, "y2009": 0.99545704236827348}, {"neighbors": [4, 31, 3, 29, 12], "y1995": 0.85570268988941878, "y1997": 0.85986131704895119, "y1996": 0.85575915188345031, "y1999": 0.85380119644969055, "y1998": 0.85693406055397725, "y2006": 0.82803647591954255, "y2007": 0.81987360180979219, "y2004": 0.83998883284341452, "y2005": 0.83478547261894065, "y2002": 0.85472102128186755, "y2003": 0.84564834502399988, "y2000": 0.86191535266765262, "y2001": 0.84981450830432048, "id": 14, "y2008": 0.82265395167873867, "y2009": 0.83994039782937002}, {"neighbors": [19, 8, 17, 16, 13], "y1995": 0.87022046646521634, "y1997": 0.85961813213722393, "y1996": 0.85996258309339635, "y1999": 0.8394713575455558, "y1998": 0.85689572413110093, "y2006": 0.94202108334913126, "y2007": 0.94222309998743192, "y2004": 0.86763340229291142, "y2005": 0.89179316746010362, "y2002": 0.86776297543511893, "y2003": 0.86720209304280604, "y2000": 0.82785596604704892, "y2001": 0.86008789452656809, "id": 15, "y2008": 0.93902708112840494, "y2009": 0.94479183757120588}, {"neighbors": [28, 26, 15, 19, 32], "y1995": 0.90134907329491731, "y1997": 0.90403990934606904, "y1996": 0.904077381347274, "y1999": 0.90399237579083946, "y1998": 0.90201769385650832, "y2006": 0.91108803862404764, "y2007": 0.90543476309316473, "y2004": 0.94338264626469681, "y2005": 0.91981795862151561, "y2002": 0.93695966482853577, "y2003": 0.94242697007039, "y2000": 0.90906631602055099, "y2001": 0.92693339421265908, "id": 16, "y2008": 0.91737137682250491, "y2009": 0.94793657442067902}, {"neighbors": [13, 18, 11, 19, 8], "y1995": 1.1977611005602815, "y1997": 1.1843915817489725, "y1996": 1.1822256425225894, "y1999": 1.1928672308275252, "y1998": 1.1826786457339149, "y2006": 1.2392938410349985, "y2007": 1.2341867605077472, "y2004": 1.2385704217423759, "y2005": 1.2441989281116201, "y2002": 1.2262477774195681, "y2003": 1.2239707531714479, "y2000": 1.2017286912636342, "y2001": 1.2132869128474402, "id": 17, "y2008": 1.2362673914436095, "y2009": 1.2675439750795283}, {"neighbors": [13, 17, 11, 8, 19], "y1995": 1.2491967813733067, "y1997": 1.2699116090397236, "y1996": 1.2575477330927329, "y1999": 1.3062566740535762, "y1998": 1.2802065055312271, "y2006": 1.3210776560048689, "y2007": 1.329362443219563, "y2004": 1.3054484140490119, "y2005": 1.3030330249408666, "y2002": 1.3257518058685978, "y2003": 1.3079549159235695, "y2000": 1.3479002255103918, "y2001": 1.3439986302151703, "id": 18, "y2008": 1.3300124123891741, "y2009": 1.3328846185074705}, {"neighbors": [26, 17, 28, 15, 16], "y1995": 1.0676800411188558, "y1997": 1.0363730321443168, "y1996": 1.0379927554499979, "y1999": 1.0329609259280523, "y1998": 1.027684488045026, "y2006": 0.94241549375546196, "y2007": 0.92754546923532677, "y2004": 0.99614160423102482, "y2005": 0.97356208269708677, "y2002": 1.0274762326434594, "y2003": 1.0316273366809443, "y2000": 1.0505901631347052, "y2001": 1.0340505678899605, "id": 19, "y2008": 0.92549226593721745, "y2009": 0.92138101880290568}, {"neighbors": [30, 25, 24, 37, 47], "y1995": 1.0947561397632881, "y1997": 1.1165429913770684, "y1996": 1.1152679554712275, "y1999": 1.1314326394231322, "y1998": 1.1310394841195361, "y2006": 1.1090538904302065, "y2007": 1.1057776900012568, "y2004": 1.1402994437897009, "y2005": 1.1197940058085571, "y2002": 1.133670175399079, "y2003": 1.139822558851451, "y2000": 1.1388962186541665, "y2001": 1.1244221220249986, "id": 20, "y2008": 1.1116682481010467, "y2009": 1.0998515545336902}, {"neighbors": [23, 22, 7, 10, 34], "y1995": 0.76530058421804126, "y1997": 0.76542450966153397, "y1996": 0.76612841163904621, "y1999": 0.76014283909933289, "y1998": 0.7672268310234307, "y2006": 0.76842416021983684, "y2007": 0.77487117798086069, "y2004": 0.76533287692895391, "y2005": 0.78205934309410463, "y2002": 0.76156903267949927, "y2003": 0.76651951668098528, "y2000": 0.74480073263159763, "y2001": 0.76098396210261965, "id": 21, "y2008": 0.77768682781054099, "y2009": 0.78801192267396702}, {"neighbors": [21, 34, 5, 7, 29], "y1995": 0.98391336093764348, "y1997": 0.98295341320156315, "y1996": 0.98075815675295552, "y1999": 0.96913802803963667, "y1998": 0.97386015032669815, "y2006": 0.93965462091114671, "y2007": 0.93069644684632924, "y2004": 0.9635616201227476, "y2005": 0.94745351657235244, "y2002": 0.97209860866113018, "y2003": 0.97441312580606143, "y2000": 0.97370819354423843, "y2001": 0.96419154157867693, "id": 22, "y2008": 0.94020973488297466, "y2009": 0.94358232339833159}, {"neighbors": [21, 10, 22, 34, 7], "y1995": 0.83561828119099946, "y1997": 0.81738501913392403, "y1996": 0.82298088022609361, "y1999": 0.80904800725677739, "y1998": 0.81748588141426259, "y2006": 0.87170334233473346, "y2007": 0.8786379876833581, "y2004": 0.85954307066870839, "y2005": 0.86790023653402792, "y2002": 0.83451612857812574, "y2003": 0.85175031934895873, "y2000": 0.80071489233375537, "y2001": 0.83358255807316928, "id": 23, "y2008": 0.87497981001981484, "y2009": 0.87888675419592222}, {"neighbors": [27, 20, 30, 32, 47], "y1995": 0.98845573274970278, "y1997": 0.99665282989553183, "y1996": 1.0209242772035507, "y1999": 0.99386618594343845, "y1998": 0.99141823200404444, "y2006": 0.97906748937234156, "y2007": 0.9932312332800689, "y2004": 1.0111665058188304, "y2005": 0.9998802359352077, "y2002": 0.99669586934394627, "y2003": 1.0255909749831356, "y2000": 0.98733194819247994, "y2001": 0.99644997431653437, "id": 24, "y2008": 1.0020493856497013, "y2009": 0.99602148231561483}, {"neighbors": [20, 33, 6, 30, 12], "y1995": 1.1493091345649815, "y1997": 1.143009615936718, "y1996": 1.1524194939429724, "y1999": 1.1398468268822266, "y1998": 1.1426554202510555, "y2006": 1.0889107875354573, "y2007": 1.0860369499254896, "y2004": 1.0856975145267398, "y2005": 1.1244348633192611, "y2002": 1.0423089214343333, "y2003": 1.0557727834721793, "y2000": 1.0831239730629278, "y2001": 1.0519262599166714, "id": 25, "y2008": 1.0599731384290745, "y2009": 1.0216094265950888}, {"neighbors": [28, 19, 16, 32, 17], "y1995": 1.1136826889802023, "y1997": 1.1189343096757198, "y1996": 1.1057147027213501, "y1999": 1.1432271991365353, "y1998": 1.1377866945457653, "y2006": 1.1268023587150906, "y2007": 1.1235793669317915, "y2004": 1.1482023546040769, "y2005": 1.1238659840114973, "y2002": 1.1600919581655105, "y2003": 1.1446778932605579, "y2000": 1.1825702862895446, "y2001": 1.1622624279436105, "id": 26, "y2008": 1.115925801617498, "y2009": 1.1257082797404696}, {"neighbors": [32, 24, 36, 16, 28], "y1995": 1.303794309231981, "y1997": 1.3120636604057812, "y1996": 1.3075218596998686, "y1999": 1.3062566740535762, "y1998": 1.3153226688859194, "y2006": 1.2865667454509278, "y2007": 1.2973409698906584, "y2004": 1.2683078569016086, "y2005": 1.2617743046198988, "y2002": 1.2920319347677043, "y2003": 1.2718351646774422, "y2000": 1.3121023910310281, "y2001": 1.2998915587009874, "id": 27, "y2008": 1.2939020510829768, "y2009": 1.2934544564717687}, {"neighbors": [26, 16, 19, 32, 27], "y1995": 0.83953719020532513, "y1997": 0.82006005316292385, "y1996": 0.82701447583159737, "y1999": 0.80294863992835086, "y1998": 0.8118887636743225, "y2006": 0.8389109342655191, "y2007": 0.84349246817602375, "y2004": 0.83108634437662732, "y2005": 0.84373783646216949, "y2002": 0.82596790474192727, "y2003": 0.82435704751379402, "y2000": 0.78772975118465016, "y2001": 0.82848010958278628, "id": 28, "y2008": 0.85637272428125033, "y2009": 0.86539395164519117}, {"neighbors": [5, 39, 22, 14, 31], "y1995": 1.2345008725695852, "y1997": 1.2353793515744536, "y1996": 1.2426021999018138, "y1999": 1.2452262575926329, "y1998": 1.2358129278404693, "y2006": 1.2365329681906834, "y2007": 1.2796200872578414, "y2004": 1.1967443443492951, "y2005": 1.2153657295128597, "y2002": 1.1937780418204111, "y2003": 1.1835533748469893, "y2000": 1.2256766974812463, "y2001": 1.2112664802237314, "id": 29, "y2008": 1.2796839248335934, "y2009": 1.2590773758694083}, {"neighbors": [37, 20, 24, 25, 27], "y1995": 0.97696620404861145, "y1997": 0.98035944080980575, "y1996": 0.9740071914763756, "y1999": 0.95543282313901556, "y1998": 0.97581530789338955, "y2006": 0.92100464312607799, "y2007": 0.9147530387633086, "y2004": 0.9298883479571457, "y2005": 0.93442917452618346, "y2002": 0.93679072759857129, "y2003": 0.92540049332494034, "y2000": 0.96480308308405971, "y2001": 0.9468637634838194, "id": 30, "y2008": 0.90249622070947177, "y2009": 0.90213630440783921}, {"neighbors": [35, 14, 33, 12, 4], "y1995": 0.84986885942491119, "y1997": 0.84295996568390696, "y1996": 0.89868510090623221, "y1999": 0.85659367787716301, "y1998": 0.87280533962476625, "y2006": 0.92562487931452408, "y2007": 0.96635366357254426, "y2004": 0.92698332540482575, "y2005": 0.94745351657235244, "y2002": 0.90448992922937876, "y2003": 0.95495898185605821, "y2000": 0.88937573313051443, "y2001": 0.89440100450887505, "id": 31, "y2008": 1.025203118044723, "y2009": 1.0394296020754366}, {"neighbors": [36, 27, 28, 16, 26], "y1995": 1.0192280751235561, "y1997": 1.0097442843101825, "y1996": 1.0025820319237864, "y1999": 0.99765073314119712, "y1998": 1.0030341681355639, "y2006": 0.94779637858468868, "y2007": 0.93759089358493275, "y2004": 0.97583768316642261, "y2005": 0.96101679691008712, "y2002": 0.99747298060178258, "y2003": 0.99550758543481688, "y2000": 1.0075901875261932, "y2001": 0.99192968437874551, "id": 32, "y2008": 0.93353431146829191, "y2009": 0.94121705123804411}, {"neighbors": [44, 25, 12, 35, 31], "y1995": 0.86367410708901315, "y1997": 0.85544345781923936, "y1996": 0.85558931627900803, "y1999": 0.84336613427334628, "y1998": 0.85103025143102673, "y2006": 0.89455097373003656, "y2007": 0.88283929116469462, "y2004": 0.85951183386707053, "y2005": 0.87194227372077004, "y2002": 0.84667960913556228, "y2003": 0.84374557883664714, "y2000": 0.83434853662160158, "y2001": 0.85813595114434105, "id": 33, "y2008": 0.90349490610221961, "y2009": 0.9060067497610369}, {"neighbors": [22, 39, 21, 29, 23], "y1995": 1.0094753356447226, "y1997": 1.0069881886439402, "y1996": 1.0041105523637666, "y1999": 0.99291086334982948, "y1998": 0.99513686502304577, "y2006": 0.96382634438484593, "y2007": 0.95011400973122428, "y2004": 0.975119236728752, "y2005": 0.96134614808826613, "y2002": 0.99291167539274383, "y2003": 0.98983209318633369, "y2000": 1.0058162611397035, "y2001": 0.98850522230466298, "id": 34, "y2008": 0.94346860300667812, "y2009": 0.9463776450423077}, {"neighbors": [31, 38, 44, 33, 14], "y1995": 1.0571257066143651, "y1997": 1.0575301194645879, "y1996": 1.0545941857842291, "y1999": 1.0510385688532684, "y1998": 1.0488078570498685, "y2006": 1.0247627521629479, "y2007": 1.0234752320591773, "y2004": 1.0329697933620496, "y2005": 1.0219168238570018, "y2002": 1.0420048344203974, "y2003": 1.0402553971511816, "y2000": 1.0480002306104303, "y2001": 1.030249414987729, "id": 35, "y2008": 1.0251768368501768, "y2009": 1.0435957064486703}, {"neighbors": [32, 43, 27, 28, 42], "y1995": 1.070841888164505, "y1997": 1.0793762307014196, "y1996": 1.0666949726007404, "y1999": 1.0794043012481198, "y1998": 1.0738798776109699, "y2006": 1.087727556316465, "y2007": 1.0885954360198933, "y2004": 1.1032213602455734, "y2005": 1.0916793915985508, "y2002": 1.0938347765734742, "y2003": 1.1052447043433509, "y2000": 1.0531800956589803, "y2001": 1.0745277096056161, "id": 36, "y2008": 1.0917733838297285, "y2009": 1.1096083021948762}, {"neighbors": [30, 40, 20, 42, 41], "y1995": 0.8671922185905101, "y1997": 0.86675155621455668, "y1996": 0.86628895935887062, "y1999": 0.86511809486628932, "y1998": 0.86425631732335095, "y2006": 0.84488343470424199, "y2007": 0.83374328958471722, "y2004": 0.84517414191529749, "y2005": 0.84843857600526962, "y2002": 0.85411284725399572, "y2003": 0.84886336375435456, "y2000": 0.86287327291635718, "y2001": 0.8516979624450659, "id": 37, "y2008": 0.82812044014430564, "y2009": 0.82878598934619596}, {"neighbors": [35, 31, 45, 39, 44], "y1995": 0.8838921149583755, "y1997": 0.90282398478743275, "y1996": 0.92288667453925455, "y1999": 0.92023285988219217, "y1998": 0.91229185518735723, "y2006": 0.93869676706720051, "y2007": 0.96947770975097391, "y2004": 0.99223700402629367, "y2005": 0.97984969609868555, "y2002": 0.93682451504456421, "y2003": 0.98655146182882891, "y2000": 0.92652175166361039, "y2001": 0.94278865361566122, "id": 38, "y2008": 1.0036262573224608, "y2009": 0.98102350657197357}, {"neighbors": [29, 34, 38, 22, 35], "y1995": 0.970820642185237, "y1997": 0.94534081352108112, "y1996": 0.95320232993219844, "y1999": 0.93967000034446724, "y1998": 0.94215592860799646, "y2006": 0.91035556215514757, "y2007": 0.90430364292511256, "y2004": 0.92879505989982103, "y2005": 0.9211054223180335, "y2002": 0.93412151936513388, "y2003": 0.93501274320242933, "y2000": 0.93092108910210503, "y2001": 0.92662519262599163, "id": 39, "y2008": 0.89994694483851023, "y2009": 0.9007386435858511}, {"neighbors": [41, 37, 42, 30, 45], "y1995": 0.95861858457245008, "y1997": 0.98254810501535106, "y1996": 0.95774543235102894, "y1999": 0.98684823919808018, "y1998": 0.98919471947721893, "y2006": 0.97163003599581876, "y2007": 0.97007020126757271, "y2004": 0.9493488753775261, "y2005": 0.97152609359561659, "y2002": 0.95601578436851964, "y2003": 0.94905384541254967, "y2000": 0.98882204635713133, "y2001": 0.97662233890759653, "id": 40, "y2008": 0.97158948117089283, "y2009": 0.95884908006927827}, {"neighbors": [40, 45, 44, 37, 42], "y1995": 0.83980438854721107, "y1997": 0.85746999875029983, "y1996": 0.84726737166133714, "y1999": 0.85567509846023126, "y1998": 0.85467221160427542, "y2006": 0.8333891885768886, "y2007": 0.83511679264592342, "y2004": 0.81743586206088703, "y2005": 0.83550405700769481, "y2002": 0.84502402428191115, "y2003": 0.82645665158259707, "y2000": 0.84818516243622177, "y2001": 0.85265681182580899, "id": 41, "y2008": 0.82136617314598481, "y2009": 0.80921873783836296}, {"neighbors": [43, 40, 46, 37, 36], "y1995": 0.95118156405662746, "y1997": 0.94688098462868708, "y1996": 0.9466212002600608, "y1999": 0.95124410099780687, "y1998": 0.95085829660091703, "y2006": 0.96895367966714574, "y2007": 0.9700163384024274, "y2004": 0.97583768316642261, "y2005": 0.95571723704302525, "y2002": 0.96804411514198463, "y2003": 0.97136213864358201, "y2000": 0.95440787445922959, "y2001": 0.96364362764682376, "id": 42, "y2008": 0.97082732652905901, "y2009": 0.9878236640328002}, {"neighbors": [36, 42, 32, 27, 46], "y1995": 1.0891004415267045, "y1997": 1.0849289528525252, "y1996": 1.0824896838138709, "y1999": 1.0945424900391545, "y1998": 1.0865692335830259, "y2006": 1.1450297539219478, "y2007": 1.1447474729339102, "y2004": 1.1334273474293739, "y2005": 1.1468606844516303, "y2002": 1.1229257675733433, "y2003": 1.1302103089739621, "y2000": 1.1055818811158884, "y2001": 1.1214085953998059, "id": 43, "y2008": 1.1408403740471014, "y2009": 1.1614292649793569}, {"neighbors": [33, 41, 45, 35, 40], "y1995": 1.0633603345917013, "y1997": 1.0869149629649646, "y1996": 1.0736582323828732, "y1999": 1.1166986255755473, "y1998": 1.0976484597942771, "y2006": 1.0839806574563229, "y2007": 1.0983176831786272, "y2004": 1.0927882684985315, "y2005": 1.0700320368873319, "y2002": 1.0881584856466706, "y2003": 1.0804431312806149, "y2000": 1.1185670222649935, "y2001": 1.0976428286056732, "id": 44, "y2008": 1.0929823187788443, "y2009": 1.0917612486217978}, {"neighbors": [41, 44, 40, 35, 33], "y1995": 0.79772064970019041, "y1997": 0.7858115114280021, "y1996": 0.78829195801876151, "y1999": 0.77035744221561353, "y1998": 0.77615921755360906, "y2006": 0.79949806580432425, "y2007": 0.80172181625581262, "y2004": 0.79603865293896003, "y2005": 0.78966436120841943, "y2002": 0.81437881076636964, "y2003": 0.80788827809912023, "y2000": 0.77751193519846906, "y2001": 0.79902973574567659, "id": 45, "y2008": 0.82168154748053679, "y2009": 0.85587910681858015}, {"neighbors": [42, 43, 40, 36, 37], "y1995": 1.0052446952315301, "y1997": 1.0047589936197736, "y1996": 1.0000769567582628, "y1999": 1.0063956091903872, "y1998": 1.0061394183885444, "y2006": 0.97292595590233411, "y2007": 0.96519561197191939, "y2004": 0.99030032232474696, "y2005": 0.97682565346267858, "y2002": 1.0081498135355325, "y2003": 1.0057431552702318, "y2000": 1.0016297948675874, "y2001": 0.99860738542320637, "id": 46, "y2008": 0.9617340332161447, "y2009": 0.95890283625473927}, {"neighbors": [20, 6, 24, 25, 30], "y1995": 0.95808418788867844, "y1997": 0.9654440995572009, "y1996": 0.93825679674127938, "y1999": 0.96987289157318213, "y1998": 0.95561201303757848, "y2006": 1.1704973973021624, "y2007": 1.1702515395802287, "y2004": 1.0533361880299275, "y2005": 1.0983262971945267, "y2002": 1.0078119390756035, "y2003": 1.0348423554112989, "y2000": 0.96608031008233231, "y2001": 0.99727184521431422, "id": 47, "y2008": 1.1873055260044207, "y2009": 1.1424264534188653}] From d1a267febb506e2d08c0710306c6a2417ebd8ae2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 12:51:01 -0400 Subject: [PATCH 055/183] adding number of classes options --- src/pg/sql/11_markov.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index e804d6c..84d7e2d 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -9,7 +9,6 @@ -- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', -- Array['date_1', 'date_2', 'date_3']) - CREATE OR REPLACE FUNCTION CDB_SpatialMarkov ( subquery TEXT, @@ -27,8 +26,7 @@ AS $$ from crankshaft.space_time_dynamics import spatial_markov_trend ## TODO: use named parameters or a dictionary - - return spatial_markov_trend(subquery, time_cols, permutations, geom_col, id_col, w_type, num_ngbrs) + return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- input table format: identical to above but in a predictable format From 54d35c614b47af5dfbebffba654a62dbd1a26cfc Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 12:52:34 -0400 Subject: [PATCH 056/183] adds tests passing for spatial_markov --- .../crankshaft/space_time_dynamics/markov.py | 6 +- .../test/test_space_time_dynamics.py | 70 ++++++++++--------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index 3e39d49..a04f6c0 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -9,7 +9,7 @@ import plpy import crankshaft.pysal_utils as pu def spatial_markov_trend(subquery, time_cols, num_classes = 7, - w_type = 'knn', num_ngbrs = 5, permutations = 999, + w_type = 'knn', num_ngbrs = 5, permutations = 0, geom_col = 'the_geom', id_col = 'cartodb_id'): """ Predict the trends of a unit based on: @@ -55,6 +55,7 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, ## build weight weights = pu.get_weight(query_result, w_type) + weights.transform = 'r' ## prep time data t_data = get_time_data(query_result, time_cols) @@ -62,7 +63,6 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, plpy.debug('shape of t_data %d, %d' % t_data.shape) plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape) plpy.debug('first num elements: %f' % t_data[0, 0]) - # ls = ps.lag_spatial(weights, t_data) sp_markov_result = ps.Spatial_Markov(t_data, weights, @@ -156,7 +156,7 @@ def get_prob_stats(prob_dist, unit_indices): Outputs: @param trend_up ndarray(float): sum of probabilities for upward movement (relative to the unit index of that prob) - @param trend_down ndarray(float): sum of probabilities for downard + @param trend_down ndarray(float): sum of probabilities for downward movement (relative to the unit index of that prob) @param trend ndarray(float): difference of upward and downward movements diff --git a/src/py/crankshaft/test/test_space_time_dynamics.py b/src/py/crankshaft/test/test_space_time_dynamics.py index d702858..54ffc9d 100644 --- a/src/py/crankshaft/test/test_space_time_dynamics.py +++ b/src/py/crankshaft/test/test_space_time_dynamics.py @@ -58,39 +58,39 @@ class SpaceTimeTests(unittest.TestCase): [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] ) - # def test_spatial_markov(self): - # """Test Spatial Markov.""" - # data = [ { 'id': d['id'], - # 'attr1': d['y1995'], - # 'attr2': d['y1996'], - # 'attr3': d['y1997'], - # 'attr4': d['y1998'], - # 'attr5': d['y1999'], - # 'attr6': d['y2000'], - # 'attr7': d['y2001'], - # 'attr8': d['y2002'], - # 'attr9': d['y2003'], - # 'attr10': d['y2004'], - # 'attr11': d['y2005'], - # 'attr12': d['y2006'], - # 'attr13': d['y2007'], - # 'attr14': d['y2008'], - # 'attr15': d['y2009'], - # 'neighbors': d['neighbors'] } for d in self.neighbors_data] - # print(str(data[0])) - # plpy._define_result('select', data) - # random_seeds.set_random_seeds(1234) - # - # result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 7, 'knn', 5, 99, 'the_geom', 'cartodb_id') - # - # print 'result == None? ', result == None - # result = [(row[0], row[1], row[2], row[3], row[4]) for row in result] - # print result[0] - # expected = self.markov_data - # for ([res_trend, res_up, res_down, res_vol, res_id], - # [exp_trend, exp_up, exp_down, exp_vol, exp_id] - # ) in zip(result, expected): - # self.assertAlmostEqual(res_trend, exp_trend) + def test_spatial_markov(self): + """Test Spatial Markov.""" + data = [ { 'id': d['id'], + 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'], + 'neighbors': d['neighbors'] } for d in self.neighbors_data] + print(str(data[0])) + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + + result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id') + + self.assertTrue(result != None) + result = [(row[0], row[1], row[2], row[3], row[4]) for row in result] + print result[0] + expected = self.markov_data + for ([res_trend, res_up, res_down, res_vol, res_id], + [exp_trend, exp_up, exp_down, exp_vol, exp_id] + ) in zip(result, expected): + self.assertAlmostEqual(res_trend, exp_trend) def test_get_time_data(self): """Test get_time_data""" @@ -240,7 +240,7 @@ class SpaceTimeTests(unittest.TestCase): [ 0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824, 0.98882205 , 0.97662234, 0.95601578, 0.94905385, 0.94934888, 0.97152609, 0.97163004 , 0.9700702, 0.97158948, 0.95884908], - [ 0.83980439, 0.84726737, 0.85747, 0.85467221, 0.8556751, 0.84818516 + [ 0.83980439, 0.84726737, 0.85747, 0.85467221, 0.8556751, 0.84818516 , 0.85265681, 0.84502402, 0.82645665, 0.81743586, 0.83550406, 0.83338919 , 0.83511679, 0.82136617, 0.80921874], [ 0.95118156, 0.9466212, 0.94688098, 0.9508583, 0.9512441, 0.95440787 @@ -263,6 +263,8 @@ class SpaceTimeTests(unittest.TestCase): , 1.17025154, 1.18730553, 1.14242645]]) self.assertTrue(np.allclose(result, expected)) + self.assertTrue(type(result) == type(expected)) + self.assertTrue(result.shape == expected.shape) def test_rebin_data(self): """Test rebin_data""" From a5cb85784152c0bf46cafa224b751ad8322db8b0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 14:17:33 -0400 Subject: [PATCH 057/183] adds docs for spatial markov --- doc/04_markov.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/04_markov.md diff --git a/doc/04_markov.md b/doc/04_markov.md new file mode 100644 index 0000000..880b632 --- /dev/null +++ b/doc/04_markov.md @@ -0,0 +1,46 @@ +## Spatial Markov + +### CDB_SpatialMarkov(subquery text, column_names text array) + +This function takes time series data associated with geometries and outputs likelihoods that the next value of a geometry will move up, down, or stay static as compared to the most recent measurement. For more information, read about [Spatial Dynamics in PySAL](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/dynamics.html). + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM real_estate_history`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| column_names | TEXT Array | Names of column that form the history of measurements for the geometries (e.g., `Array['y2011', 'y2012', 'y2013', 'y2014', 'y2015', 'y2016']`). | +| num_classes (optional) | INT | Number of quantile classes to separate data into. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | +| permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | +| geom_col (optional) | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col (optional) | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| trend | NUMERIC | | +| trend_up | NUMERIC | | +| trend_down | NUMERIC | The statistical significance (from 0 to 1) of a cluster or outlier classification. Lower numbers are more significant. | +| volatility | NUMERIC | A measure of the variance of the probabilities returned from the Spatial Markov predictions | +| rowid | NUMERIC | id of the row that corresponds to the `id_col` (by default `cartodb_id` of the input rows) | + + +#### Example Usage + +```sql +SELECT + c.the_geom, + m.trend, + m.trend_up, + m.trend_down, + m.volatility +FROM CDB_SpatialMarkov('SELECT * FROM nyc_real_estate' + Array['m03y2009','m03y2010','m03y2011','m03y2012','m03y2013','m03y2014','m03y2015','m03y2016']) As m +JOIN nyc_real_estate As c +ON c.cartodb_id = m.rowid; +``` From edf635886b56a30579272c0dca1c614f1bc9287f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 14:19:13 -0400 Subject: [PATCH 058/183] updates column name to rowid --- src/pg/sql/11_markov.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 84d7e2d..18d28b6 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -19,7 +19,7 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') -RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, ids INT) +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') From c0dfaa8341e3c35b97e393eac804a8c702872afa Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 2 Jun 2016 16:19:15 -0400 Subject: [PATCH 059/183] adds test for spatial markov sql function --- src/pg/test/sql/05_markov_test.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/pg/test/sql/05_markov_test.sql diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql new file mode 100644 index 0000000..d91fc0a --- /dev/null +++ b/src/pg/test/sql/05_markov_test.sql @@ -0,0 +1,15 @@ +SET client_min_messages TO WARNING; +\set ECHO none +\pset format unaligned +\i test/fixtures/markov_usjoin_example.sql + +-- Areas of Interest functions perform some nondeterministic computations +-- (to estimate the significance); we will set the seeds for the RNGs +-- that affect those results to have repeatable results +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility + FROM markov_usjoin_example As m1 + JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id') As m2 + ON m1.cartodb_id = m2.rowid + ORDER BY m1.cartodb_id; From fc3a7e3d7868069df402106651869820dcc35b5c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 3 Jun 2016 11:01:20 -0400 Subject: [PATCH 060/183] add casting to function call --- src/pg/test/sql/05_markov_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index d91fc0a..7b876e3 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,6 +10,6 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility FROM markov_usjoin_example As m1 - JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id') As m2 + JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 ON m1.cartodb_id = m2.rowid ORDER BY m1.cartodb_id; From 183a0f960497ed74e532ec2a3b8a481159d19d8d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 3 Jun 2016 11:01:53 -0400 Subject: [PATCH 061/183] comment out unneeded code whch w.transform handles --- .../crankshaft/pysal_utils/pysal_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 6cea5f0..c21a834 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -22,13 +22,13 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5): Construct PySAL weight from return value of query @param query_res dict-like: query results with attributes and neighbors """ - if w_type.lower() == 'knn': - row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs - weights = {x['id']: row_normed_weights for x in query_res} - else: - weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - if len(x['neighbors']) > 0 - else [] for x in query_res} + # if w_type.lower() == 'knn': + # row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + # weights = {x['id']: row_normed_weights for x in query_res} + # else: + # weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + # if len(x['neighbors']) > 0 + # else [] for x in query_res} neighbors = {x['id']: x['neighbors'] for x in query_res} print 'len of neighbors: %d' % len(neighbors) From 5e8d11fce22107fe47b77e0aa563f656213f1fcd Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 3 Jun 2016 11:02:51 -0400 Subject: [PATCH 062/183] adds fixtures for spatial markov tests --- .../test/fixtures/markov_usjoin_example.sql | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/pg/test/fixtures/markov_usjoin_example.sql diff --git a/src/pg/test/fixtures/markov_usjoin_example.sql b/src/pg/test/fixtures/markov_usjoin_example.sql new file mode 100644 index 0000000..350bb5b --- /dev/null +++ b/src/pg/test/fixtures/markov_usjoin_example.sql @@ -0,0 +1,180 @@ +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: markov_usjoin_example; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE markov_usjoin_example ( + cartodb_id integer NOT NULL, + the_geom geometry(Geometry,4326), + name text, + state_fips text, + y1929 numeric, + y1930 numeric, + y1931 numeric, + y1932 numeric, + y1933 numeric, + y1934 numeric, + y1935 numeric, + y1936 numeric, + y1937 numeric, + y1938 numeric, + y1939 numeric, + y1940 numeric, + y1941 numeric, + y1942 numeric, + y1943 numeric, + y1944 numeric, + y1945 numeric, + y1946 numeric, + y1947 numeric, + y1948 numeric, + y1949 numeric, + y1950 numeric, + y1951 numeric, + y1952 numeric, + y1953 numeric, + y1954 numeric, + y1955 numeric, + y1956 numeric, + y1957 numeric, + y1958 numeric, + y1959 numeric, + y1960 numeric, + y1961 numeric, + y1962 numeric, + y1963 numeric, + y1964 numeric, + y1965 numeric, + y1966 numeric, + y1967 numeric, + y1968 numeric, + y1969 numeric, + y1970 numeric, + y1971 numeric, + y1972 numeric, + y1973 numeric, + y1974 numeric, + y1975 numeric, + y1976 numeric, + y1977 numeric, + y1978 numeric, + y1979 numeric, + y1980 numeric, + y1981 numeric, + y1982 numeric, + y1983 numeric, + y1984 numeric, + y1985 numeric, + y1986 numeric, + y1987 numeric, + y1988 numeric, + y1989 numeric, + y1990 numeric, + y1991 numeric, + y1992 numeric, + y1993 numeric, + y1994 numeric, + y1995 numeric, + y1996 numeric, + y1997 numeric, + y1998 numeric, + y1999 numeric, + y2000 numeric, + y2001 numeric, + y2002 numeric, + y2003 numeric, + y2004 numeric, + y2005 numeric, + y2006 numeric, + y2007 numeric, + y2008 numeric, + y2009 numeric +); + + +ALTER TABLE public.markov_usjoin_example OWNER TO postgres; + +-- +-- Data for Name: markov_usjoin_example; Type: TABLE DATA; Schema: public; Owner: postgres +-- + +COPY markov_usjoin_example (cartodb_id, the_geom, name, state_fips, y1929, y1930, y1931, y1932, y1933, y1934, y1935, y1936, y1937, y1938, y1939, y1940, y1941, y1942, y1943, y1944, y1945, y1946, y1947, y1948, y1949, y1950, y1951, y1952, y1953, y1954, y1955, y1956, y1957, y1958, y1959, y1960, y1961, y1962, y1963, y1964, y1965, y1966, y1967, y1968, y1969, y1970, y1971, y1972, y1973, y1974, y1975, y1976, y1977, y1978, y1979, y1980, y1981, y1982, y1983, y1984, y1985, y1986, y1987, y1988, y1989, y1990, y1991, y1992, y1993, y1994, y1995, y1996, y1997, y1998, y1999, y2000, y2001, y2002, y2003, y2004, y2005, y2006, y2007, y2008, y2009) FROM stdin; +42 0106000020E6100000010000000103000000010000002900000000000040F9825CC0FFFFFFDF9F11434000000060D5825CC0000000201B494340000000E0C7825CC0000000E0EA5643400000008083825CC000000080F5C44340FFFFFF1F7F825CC0000000E054F44340000000A062825CC0000000E0380E4440FFFFFF3F63825CC0FFFFFF9FB67F44400000004072825CC0000000C06BFF4440000000C0473F5CC00000004028004540000000605D095CC000000080E3FF4440000000E061065CC0000000E04D004540000000C09ADF5BC0000000E007004540000000A011C35BC0000000A085FF44400000008038C35BC00000002011CA44400000002042C35BC0000000E015A14440000000E037C35BC0000000C0917F4440000000A0FF835BC000000080BC7F4440000000E017805BC0000000E0B27F4440000000800C435BC000000000CE7F444000000020E9425BC00000008029554440000000E03C435BC000000020F31A4440FFFFFF1F52435BC00000000026D443400000002062435BC00000004054C24340000000803F435BC0000000E034AE4340FFFFFFFF68435BC000000020513F4340FFFFFF5F88435BC0000000E0591F434000000040BD425BC0000000A09313434000000000B9425BC0FFFFFF3F97F14240FFFFFF3FE0425BC0FFFFFFFFBED04240FFFFFF7F0F435BC000000000927F424000000000C57F5BC000000000FC7E424000000020E69C5BC0000000A0F17E424000000000F09E5BC0000000C080804240FFFFFFDF51AF5BC0000000A051804240000000C0BFD65BC0000000603880424000000000230F5CC0000000C06C7F424000000040AC225CC0000000C0BE7F4240FFFFFFBF8C395CC0000000E0847F4240000000C0C2825CC0000000608F7F424000000000F9825CC0000000E09BCC424000000040F9825CC0FFFFFFDF9F114340 Utah 49 0.47829861111111111111 0.48115942028985507246 0.41884222474460839955 0.45118343195266272189 0.47603833865814696486 0.45588235294117647059 0.53878116343490304709 0.53341013824884792627 0.46786090621707060063 0.55849056603773584906 0.50945494994438264739 0.46738072054527750730 0.51030927835051546392 0.56482670089858793325 0.70684243565599497803 0.65603502188868042527 0.68187347931873479319 0.62265224815025611838 0.66666666666666666667 0.69218061674008810573 0.69916897506925207756 0.64963855421686746988 0.69832654907281772953 0.66583229036295369212 0.65549652635880670208 0.65134575569358178054 0.65927977839335180055 0.63950527464532557294 0.66263345195729537367 0.69132186012449652142 0.69562146892655367232 0.69548872180451127820 0.68737672583826429980 0.70037688442211055276 0.70184615384615384615 0.70155836518670979124 0.69606475020932179738 0.67243159525038719670 0.64862932061978545888 0.65271213144271888364 0.64018980812873942645 0.66620825147347740668 0.69018867924528301887 0.69843777426715815341 0.69315814773273513860 0.69616908850726552180 0.71142422986600359166 0.72783766645529486367 0.72635445362718089991 0.71993865030674846626 0.70968918056694922979 0.68044054988343114398 0.67003245582401730977 0.65805542508219821512 0.65402873599594911070 0.63896473265073947668 0.63134893140755742685 0.61123864657151412317 0.57720940854076273122 0.54800683134085891615 0.54180718688219560414 0.56089168162776780371 0.58299519785578676991 0.57111925964728479134 0.57533274778731166813 0.58666317306432595310 0.59029016809090055404 0.59616993307839388145 0.59433644229688729071 0.59526861048809142369 0.59256997455470737913 0.58826279527559055118 0.58892121384138697699 0.59517860117560267485 0.59417897070616538516 0.58141704992334866360 0.59344562119858789503 0.57853049889503842919 0.57498609308362692379 0.57315507629107981221 0.57090033373786407767 +37 0106000020E6100000030000000103000000010000001E000000FFFFFF7F9AF251C000000060F4CC444000000000C2F251C0000000E021D24440000000A07AF251C0FFFFFF3F5ADC4440FFFFFF9F17F351C0000000E08800454000000000DEDF51C0000000002C014540000000E043D851C000000020BE0145400000008081D851C00000004055FD4440000000E09AD851C000000020B5F144400000008059D551C0000000E0ADF24440000000A0F3D551C0FFFFFF5F16F044400000006071D551C000000080CCED4440000000A024D651C00000006012E8444000000080C7D551C0000000C064E444400000002078D451C000000000D4E24440000000C018D151C000000040F4DF4440000000E0AFCE51C00000006092DA44400000006035D251C000000020F8D64440000000808BD751C000000040E1DE4440FFFFFF9F38D951C0000000406AE1444000000020A6D751C00000002002DA444000000020DDDA51C0FFFFFF5F78D34440FFFFFF5F61DB51C00000008048BE44400000008062DF51C0FFFFFF7F2CB24440000000A041EE51C0FFFFFF7FE0A94440000000A07FF751C0000000404DA94440FFFFFFDF49F651C0FFFFFFBFA1A944400000004097F551C000000020C2AB4440000000C02CF651C000000040AEB344400000002068F351C0000000A036B54440FFFFFF7F9AF251C000000060F4CC444001030000000100000005000000000000A0C1CC51C0000000C0D5D644400000000012C951C0000000A0DCD344400000008087C751C00000004019BF444000000020D4CC51C0FFFFFF9F4ABB4440000000A0C1CC51C0000000C0D5D64440010300000001000000060000000000006042D151C00000006082CF4440000000C013CE51C00000006059D14440000000C04ECF51C000000080C4BC44400000000077D251C0000000E0E3BD4440000000E066D651C0000000800EB944400000006042D151C00000006082CF4440 Rhode Island 44 0.75868055555555555556 0.76135265700483091787 0.80703745743473325766 0.85059171597633136095 0.89297124600638977636 0.88235294117647058824 0.89335180055401662050 0.81912442396313364055 0.77028451001053740780 0.84528301886792452830 0.80088987764182424917 0.73028237585199610516 0.80240549828178694158 0.73748395378690629012 0.75392341494036409291 0.79674796747967479675 0.77737226277372262774 0.77916903813318155948 0.82429378531073446328 0.78909691629955947137 0.76343490304709141274 0.74843373493975903614 0.77657168701944821348 0.73633708802670004172 0.75807110747854515734 0.76480331262939958592 0.77483181638306292046 0.72680974899963623136 0.72028469750889679715 0.75686561699011351153 0.77083333333333333333 0.76349965823650034176 0.76561472715318869165 0.77167085427135678392 0.78523076923076923077 0.79123787121434872096 0.80100474462740720067 0.80356220960247805885 0.79523241954707985697 0.80913796984019806437 0.79740045388900350732 0.80825147347740667976 0.81037735849056603774 0.81183078813410566965 0.79666720076910751482 0.79333626889769558198 0.80729382511396601741 0.81306277742549143944 0.80394857667584940312 0.78660531697341513292 0.78342904019688269073 0.78318192780770158373 0.78002163721601153985 0.77870227470978997517 0.78732831191847585290 0.78026166097838452787 0.78265735756542130789 0.77787204311807565625 0.76049326330212377255 0.76106968800766443121 0.76198782804202038997 0.75531119090365050868 0.75803149313181699736 0.74234328618823118561 0.74782109316938044727 0.74551290449364601074 0.75268413309543932138 0.75056763862332695985 0.74814585908529048208 0.74794403503150699562 0.74750636132315521628 0.73043799212598425197 0.74216514108659145202 0.77042431165369696104 0.79461295344120005661 0.78468750694305583328 0.77540300284972991366 0.75508966812039191911 0.74950862228815130725 0.76184712441314553991 0.78282387742718446602 +14 0106000020E6100000010000000103000000010000007C000000000000A096C457C0000000202E8042400000000002DA57C0000000A01180424000000020A6E157C0000000C01E8042400000008045F257C0000000A0BF7F4240000000004BFD57C00000006000804240000000E05E0058C0FFFFFF1FC77F4240000000A0352158C0000000A011804240FFFFFF9FE52F58C00000006004804240FFFFFFBFCA4858C0000000C0F87F4240000000E0C35D58C0000000608B7F424000000060737358C000000040D07F424000000000AB8658C0000000A0D37F4240000000E0319658C000000060E07F4240000000208DA258C0000000E0C97F4240000000E0F1BF58C0000000E0BF7F4240FFFFFF1FF9DB58C0000000C04C7F4240000000E0D4E258C0FFFFFF5F6A7F424000000020EDFF58C0000000E0687F4240FFFFFF5FA40559C000000020B27F424000000060902859C000000020B87F424000000020CF3C59C0000000C0917F4240FFFFFFDF8D4459C000000020AC7F424000000040686359C0000000C0927F4240000000608A8159C0000000C0927E4240000000405A8259C0000000A0967E4240000000E0A88259C00000000071B14240000000E0D08259C0000000601BD2424000000000C98259C00000006000DE424000000020EB8259C0FFFFFFBF7C204340000000C0E28259C0FFFFFFBFB421434000000020048359C000000040C84E4340000000E0038359C000000020A5584340000000E01A8359C000000060BC844340000000E0088359C0000000603990434000000000188359C0FFFFFF1F0AC8434000000080228359C000000020CBC84340000000C0448359C0000000E0DCFF4340FFFFFF7F0B5A59C00000000021004440000000E0965459C0000000C03B00444000000080483059C0FFFFFF5F0600444000000000042F59C0000000C0E4FF434000000060340C59C00000000013004440000000408D0B59C0000000800F0044400000008028E858C0000000A061004440000000A061CB58C0000000C0F1FF4340FFFFFFDF1EC458C000000020C9FF43400000002019AE58C000000020CDFF4340000000E043A058C000000080A1FF4340000000C0E29058C000000040CCFF434000000020797B58C0000000E0CCFF4340000000C03D7458C0000000C0F6FF434000000060245758C0FFFFFFFFA9FF4340FFFFFF5F1C3A58C0000000E081FF434000000080453358C0000000E04AFF4340000000200A1D58C00000000041FF434000000040610F58C0FFFFFFDF4BFF434000000000100058C0FFFFFF5F61FF4340FFFFFF9FF2F157C0000000A02AFF4340000000A015D557C0000000600DFF434000000080BDD357C0FFFFFF9FECFF4340000000C067CF57C00000000097F843400000002045CD57C00000004016F84340FFFFFFBF65CC57C0000000E080F44340000000809EC957C0000000403BF44340000000206EC657C0000000E057EF4340FFFFFF1F08C457C0FFFFFFDFEAEE4340FFFFFFDF20C257C0FFFFFF5F5DF04340000000A060C157C000000060D0F24340000000E0BEBD57C0FFFFFF5F4EF343400000002008BC57C000000000B3F24340000000C0EBBB57C0000000E0B8EC4340FFFFFFBF1CBB57C0FFFFFF3FA4EA4340000000207AB957C0FFFFFFFF06EA434000000040D9B857C0000000C0A0E84340FFFFFF7F8AB957C0000000A09AE54340000000A0B6BB57C00000002032E44340000000E0D4BB57C00000006041E34340000000C0FABA57C00000002001E14340000000E01DB857C0000000205EE1434000000080BDB757C00000002087E04340000000E02AB857C0000000C0A1DE4340000000A0F2B957C00000008006DD4340000000208FBB57C0000000600FDD434000000040FCBC57C0000000E045DE4340000000E089BD57C0000000A0B3DD4340FFFFFFDF9CBE57C0FFFFFFDFADD7434000000080CBC157C0000000C0B9D44340000000C091C357C0000000C016D04340000000606AC357C0000000A01BCB4340000000A0F5C657C0000000E0C4C74340FFFFFFBF83C657C00000008034C44340000000E007C357C0000000601FBE4340000000C093C257C0000000C041BB4340000000001ABF57C0FFFFFF5F40B84340FFFFFFFF53BD57C0FFFFFF5FAAB44340000000803BBB57C000000060CDB043400000008079B957C0000000E0B8B04340000000804FBA57C00000002089AB43400000008013BA57C00000000059A943400000002060B857C0000000209DA44340FFFFFFBF53B557C00000008081A143400000006084B457C000000020029B4340000000408CB257C0000000603399434000000020BDAE57C0000000A0EB954340000000C037AB57C0000000C063964340000000E05AA957C00000004047944340000000E031A757C00000002069934340000000A076A657C0000000A013924340000000E0E7A657C000000020708E4340000000A0FAA657C000000080B7854340FFFFFFDF2EA757C000000020266B4340000000003AA757C0000000005D5E43400000004095A757C0000000C0583C4340FFFFFFBF9AA757C0000000802D324340000000807EA757C0000000E022074340000000C074A757C0000000A0E2034340FFFFFF9F9EA757C00000008004D74240000000C099A757C0000000003ED342400000000096A757C0000000E02BAE42400000008099A757C000000060F1A9424000000040B5A757C0000000E0B0874240FFFFFF9FB0A757C0000000209D7F4240000000A014C257C00000004017804240000000A096C457C0000000202E804240 Kansas 20 0.46180555555555555556 0.45120772946859903382 0.45516458569807037457 0.39349112426035502959 0.39936102236421725240 0.42205882352941176471 0.50138504155124653740 0.44585253456221198157 0.45100105374077976818 0.48176100628930817610 0.42491657397107897664 0.41382667964946445959 0.47422680412371134021 0.54621309370988446727 0.65599497802887633396 0.73108192620387742339 0.70863746958637469586 0.64655663062037564030 0.74011299435028248588 0.74449339207048458150 0.71966759002770083102 0.70506024096385542169 0.72455902306648575305 0.76220275344180225282 0.71066612178177360033 0.74161490683229813665 0.69370795409576573011 0.66242269916333212077 0.68007117437722419929 0.76821677041376785060 0.74364406779661016949 0.73991797676008202324 0.73208415516107823800 0.72393216080402010050 0.73353846153846153846 0.73978241693619523670 0.75048841752721183366 0.74703149199793495096 0.72276519666269368296 0.73553905019131217646 0.73199917474726635032 0.74970530451866404715 0.78207547169811320755 0.80972441635948744953 0.84505688190995032847 0.83913107294877440188 0.85453791960215499378 0.85237793278376664553 0.83872819100091827365 0.82638036809815950920 0.84222039923434509161 0.80697805289814293754 0.81125135232600072124 0.80446889887942025096 0.78315083233116020001 0.77372013651877133106 0.76373714224804135799 0.74378680507036630402 0.71171500342543959808 0.68025992418877827300 0.66259642594100089158 0.68005685218432076601 0.70103860328332650858 0.69687445433909551248 0.69285859063576785352 0.69933184855233853007 0.68516605628071493411 0.69075645315487571702 0.68420609057197437914 0.68586457332051692833 0.68254452926208651399 0.68444881889763779528 0.68542302325031339436 0.69172556578853430428 0.71015142223689796689 0.69277255659979115288 0.68833737399515120582 0.68319871707116734790 0.67765622102725755609 0.69657790492957746479 0.70229065533980582524 +1 0106000020E6100000010000000103000000010000008F000000FFFFFF1F7D4455C0FFFFFFFF04FB3F40000000C05F4755C00000008042E83F40FFFFFF3FAE4855C000000020CFDA3F40000000A06C4855C0FFFFFF5F9DC83F40FFFFFFBF554855C00000004058C73F4000000000624755C0000000203DBB3F40FFFFFF7F994755C0FFFFFF1F5AB53F40000000201F4755C0000000A01FAF3F40FFFFFF7FCC4355C000000040009F3F4000000020BD4255C000000060E18D3F40000000C0BC4255C001000040FD843F40000000A03F4455C0010000C0D5793F4000000040EF4355C0FFFFFFFFC3703F4000000000A24555C0FFFFFF9FF75D3F40000000E0494555C0FFFFFF5F08553F4000000000344555C0000000208B4D3F40000000A09E4655C00000004071453F40000000C0FE4555C0000000E0153A3F4000000000984655C0000000C05D323F40000000E0FA4555C0000000400A2C3F4000000020664455C0000000E084293F40000000E0704255C00100006064203F40000000A0174155C0FFFFFF9F75143F40000000801B4055C00000000046003F40000000800B5F55C0000000C035003F4000000080255F55C0FFFFFF7F35003F40000000600A8255C0000000E03EFE3E40000000409E8B55C000000040BBFE3E4000000000939855C0000000A0C9FD3E40000000E0EBAC55C00000002077FF3E40000000C0E1B155C0000000A07DFF3E400000000071CA55C000000000C3003F40000000604FE655C001000080A0003F40000000A0C0E555C0FFFFFFFF44F43E40000000E00BE855C0FFFFFF7F70E03E40000000606BE755C0FFFFFFBF1DD93E40000000C0AEE255C0FFFFFFFF16C93E4000000020B4E155C0010000C090BF3E400000004073DD55C000000000A3B43E4000000080CFDA55C0FFFFFF7F4FB13E40000000C083D955C000000080F6AA3E40000000E02BD955C001000020B29E3E400000008003DB55C0000000A0698E3E400000000081DC55C0FFFFFF5FFC873E4000000040CEDA55C0000000E0437B3E4000000000F0D955C0FFFFFF7FAE703E4000000080DADD55C0FFFFFF9F095C3E40000000009BE555C0FFFFFF7FAD513E40000000A0FAE555C0FFFFFFFF38473E40000000E0E6F255C0000000A0D53B3E40000000002A0056C0FFFFFF1FC03B3E40FFFFFFBFEAF155C000000060BE453E40000000207BF055C0000000C0994C3E4000000080D2F955C000000020CD6B3E400000000075FA55C0FFFFFF5FF99E3E4000000040440156C0000000E076BE3E4000000000AF0856C0000000E042563E4000000020801456C0000000C072673E40FFFFFF9FB01956C0FFFFFFBFB2643E40000000E0B31A56C0FFFFFFFF7BBC3E40FFFFFFBF771B56C00000008021003F40000000A0CF1B56C000000020E61E3F40000000A0D91C56C0000000A0796F3F40FFFFFFBFC31D56C0000000E0BBB33F4000000060441E56C00000006083E33F4000000020031C56C0FFFFFF5F221D4040000000A03F1B56C0000000009027404000000000341956C000000060484A4040000000C0451656C0000000405A76404000000040B91556C000000020627E4040FFFFFFBF811356C000000080F6A4404000000040921156C0000000A0F3C44040FFFFFFDFE10F56C0000000000EDF404000000080F70C56C0000000E08D07414000000080C10C56C000000060900B414000000080B90A56C0000000607A29414000000080AD0956C0FFFFFFDF8A3B4140FFFFFF1FBA0856C0000000A04A4A4140000000A0C90556C000000020A172414000000060F70656C0FFFFFF3F2E73414000000000280956C0000000A011774140000000C0690C56C0FFFFFF1F8F804140000000A0790C56C0FFFFFFFFB8814140000000601BFF55C0000000A00A82414000000000E6E655C0000000E0568141400000008041CE55C000000000EE804140000000E048CD55C00000002002814140000000C056B555C0000000C0C37F41400000006012B255C0000000609D7F414000000040099455C000000040627F4140000000C06C9355C000000080687F414000000060A77755C0000000C0037F4140000000C0F96655C000000020BB7E414000000060516555C0000000005E6E4140FFFFFFFF386255C000000000AB4F4140000000E0A76155C000000040D44A4140000000E0C26055C000000060EF42414000000000765D55C00000000021254140000000C0A95A55C0FFFFFFFF1C0B414000000080545955C000000060D8FA404000000060945855C000000020E1F3404000000020765555C000000000D1D34040000000A0855355C000000020C8BE4040000000E0CE5255C0000000607FB64040FFFFFF3FF84E55C0FFFFFFDF86904040FFFFFF9FF24E55C0000000C05C8F4040000000E0914B55C000000060936F404000000020644A55C00000002056674040000000A0334855C0000000C07263404000000000894855C0000000A0CA60404000000020FF4755C0000000A0435F4040000000A04A4755C000000020FC5D404000000020E64655C0000000A04C58404000000060CB4555C0000000C087564040000000C0A54655C0000000C0A852404000000060874555C0000000A06C504040000000C0634555C0000000C02A4D4040000000A0884455C000000000624A4040000000E0B93F55C0000000A067424040000000405A3F55C000000060333A404000000000CA3D55C000000080F3364040FFFFFF5F253E55C000000000C632404000000000033F55C00000008087314040000000E0303E55C0000000A0882F404000000060234055C000000040682C404000000080584055C0000000402B2A404000000000F43A55C0000000A08025404000000060423955C00000000061224040000000C0383955C0000000C027214040000000A0F23955C000000080EC1F4040000000C01E3B55C000000080A51F4040FFFFFFFFA03A55C0000000603C1D4040000000A06B3B55C0000000C0E11B4040000000C0733E55C000000000261B404000000020783D55C0000000208C18404000000060814055C000000080E0164040000000A0904155C00000008043154040000000A06A4355C0000000A030104040000000E0F54255C0000000609D0B404000000060A14355C000000000E508404000000020F94355C0000000A06106404000000020A24355C0FFFFFF3F34024040FFFFFF1F7D4455C0FFFFFFFF04FB3F40 Alabama 01 0.28038194444444444444 0.25797101449275362319 0.25425652667423382520 0.23964497041420118343 0.26517571884984025559 0.31029411764705882353 0.30055401662049861496 0.28917050691244239631 0.28134878819810326660 0.30691823899371069182 0.28031145717463848721 0.27361246348588120740 0.32216494845360824742 0.33247753530166880616 0.41305712492153170119 0.46153846153846153846 0.47688564476885644769 0.42914058053500284576 0.45480225988700564972 0.48513215859030837004 0.46149584487534626039 0.43807228915662650602 0.47263681592039800995 0.46141009595327492699 0.47445852063751532489 0.47163561076604554865 0.50375939849624060150 0.49327028010185522008 0.50569395017793594306 0.53753203954595386305 0.53884180790960451977 0.53246753246753246753 0.52169625246548323471 0.52355527638190954774 0.54092307692307692308 0.55571890620405763011 0.56656433156572704438 0.55988642230252968508 0.54684147794994040524 0.56628404231375196939 0.56694862801733030741 0.58526522593320235756 0.60849056603773584906 0.62208179743724767421 0.63451369972760775517 0.63863202700719213269 0.65824008841000138141 0.67507926442612555485 0.66769972451790633609 0.66462167689161554192 0.65618448637316561845 0.63445614599244312244 0.62834475297511720159 0.61631886197409917466 0.61921640610165200329 0.61433447098976109215 0.61733198315834354847 0.60894300828425990618 0.58972368120575473852 0.57658183029949598034 0.57754777687327983874 0.59216038300418910832 0.61556788147265755872 0.60981316570630347477 0.60776298898723059253 0.61771256386741779117 0.61611418912573950606 0.60734345124282982792 0.59357793010450612428 0.59070276620741215422 0.58491094147582697201 0.57753444881889763780 0.57870337519808888574 0.59877204255015349468 0.61476956460210387282 0.61465484680841609456 0.61879545744545106546 0.59910429663817886688 0.59313925458928240312 0.60187426643192488263 0.61199180825242718447 +29 0106000020E6100000010000000103000000010000003A000000000000E020435BC00000006092384040000000203F435BC000000000C6634040FFFFFF9F31435BC0000000C0409A4040000000A02E435BC00000008041E44040FFFFFF9F12435BC000000020BE4B4140000000A0F1425BC0000000C0307A4140FFFFFF3F05435BC00000006092FF4140FFFFFF7F0F435BC000000000927F424000000020CC175BC0000000C0EE7F4240FFFFFFDF32DE5AC0000000C0D77F42400000000041DA5AC0000000C0AE7F424000000020F2B85AC0000000A0E17F42400000000015B75AC0000000A0A77E4240000000802E9E5AC000000040E97E424000000080737F5AC0000000E0027F4240FFFFFFFF9F6D5AC0FFFFFF1F4D7F4240000000209A4D5AC0000000E00C7F4240000000C051495AC0000000A0207F4240FFFFFF1F8FFF59C0000000004A7F4240000000C0F3C459C000000080F77F4240FFFFFF7FD2BF59C000000000CF7F424000000080CDBF59C000000020053F424000000020B7C159C0000000A0EB3E4240FFFFFFFF81C159C0000000E02B074240FFFFFF5F6AC159C00000004003DF4140FFFFFF1F65C159C000000040D2CF414000000040A4C159C0000000C0AE9641400000008095C159C0000000E07B7B4140000000006BC159C0000000E0645F414000000080DDC159C00000002064274140000000C018C259C000000040BDE940400000008072C259C0000000406AC84040FFFFFFFFB9C259C00000006059B040400000002020C359C0000000600D7A4040000000A0CFC359C0000000A0F9414040FFFFFF3F88C359C000000020E10A4040000000C0B5C359C0FFFFFF3F3E004040FFFFFFFF40D559C0000000208800404000000080A7EE59C000000000C800404000000000C7FE59C000000000C10040400000004034015AC0FFFFFF7FEE004040000000A06F365AC00000004067004040000000C0FE3A5AC0000000E08B004040000000602C805AC0000000E032004040000000A02E985AC00000002015004040FFFFFF7FE0A75AC00000006020004040000000A091A95AC000000040F0FA3F40000000A073A85AC0000000C0DCF83F400000006086A85AC0000000C0FBE93F40000000A02FA95AC00000006025E53F40000000A065A75AC0FFFFFFFF39D83F400000000053A75AC0000000A056D13F40FFFFFF7F7EA25AC00000004044C93F40000000A01CD25AC000000080F4C83F4000000060F80C5BC0000000006CC93F40FFFFFF9F710D5BC0FFFFFF7FFF573F4000000060E1425BC0FFFFFF9FE5573F40000000E020435BC00000006092384040 New Mexico 35 0.35590277777777777778 0.32270531400966183575 0.32803632236095346198 0.30769230769230769231 0.33706070287539936102 0.36323529411764705882 0.40443213296398891967 0.39516129032258064516 0.38145416227608008430 0.42515723270440251572 0.39710789766407119021 0.36806231742940603700 0.40721649484536082474 0.40821566110397946085 0.48524795982423101067 0.55096935584740462789 0.57299270072992700730 0.52874217416050085373 0.57231638418079096045 0.61894273127753303965 0.63434903047091412742 0.58024096385542168675 0.60877431026684758028 0.59365874009178139341 0.59011033919084593380 0.60538302277432712215 0.61060546102097348635 0.59148781375045471080 0.61672597864768683274 0.66825338703771512267 0.66913841807909604520 0.64388243335611756664 0.64168310322156476003 0.63002512562814070352 0.63200000000000000000 0.62746251102616877389 0.62796539212950041864 0.61590087764584408880 0.59356376638855780691 0.60972316002700877785 0.60264080874767897669 0.62809430255402750491 0.64735849056603773585 0.66017202036159382131 0.66287453933664476847 0.67048290033758990166 0.69691946401436662522 0.70095117311350665821 0.69869146005509641873 0.70010224948875255624 0.69446723179290857716 0.67545622638475761717 0.67320591417237648756 0.66389317587063007448 0.65618077093486929553 0.63794084186575654152 0.63950327772744230667 0.61014073260804471504 0.57940168988353505367 0.55492148123463989670 0.54599372020002325852 0.55954518252543387193 0.58608494955887279902 0.57359874279727606076 0.58192014053104519965 0.58777675881042840299 0.59010235702882899803 0.58191921606118546845 0.56840656253511630520 0.56547046886681619139 0.55605597964376590331 0.54633366141732283465 0.57222261642896000378 0.58175674067728040742 0.59266946554082739752 0.59112621920060432358 0.59929394751392964995 0.58236363991942581112 0.58075282773966252550 0.59758289319248826291 0.61053170509708737864 +5 0106000020E61000000100000001030000000100000038000000000000E0D08259C0000000601BD24240000000E0A88259C00000000071B14240000000405A8259C0000000A0967E4240FFFFFF7FD2BF59C000000000CF7F4240000000C0F3C459C000000080F77F4240FFFFFF1F8FFF59C0000000004A7F4240000000C051495AC0000000A0207F4240000000209A4D5AC0000000E00C7F4240FFFFFFFF9F6D5AC0FFFFFF1F4D7F424000000080737F5AC0000000E0027F4240000000802E9E5AC000000040E97E42400000000015B75AC0000000A0A77E424000000020F2B85AC0000000A0E17F42400000000041DA5AC0000000C0AE7F4240FFFFFFDF32DE5AC0000000C0D77F424000000020CC175BC0000000C0EE7F4240FFFFFF7F0F435BC000000000927F4240FFFFFF3FE0425BC0FFFFFFFFBED0424000000000B9425BC0FFFFFF3F97F1424000000040BD425BC0000000A093134340FFFFFF5F88435BC0000000E0591F4340FFFFFFFF68435BC000000020513F4340000000803F435BC0000000E034AE43400000002062435BC00000004054C24340FFFFFF1F52435BC00000000026D44340000000E03C435BC000000020F31A444000000020E9425BC00000008029554440000000800C435BC000000000CE7F444000000020C1FA5AC0000000C06F804440000000806BD35AC000000080058044400000008059B75AC000000080CE7F4440000000E006955AC0000000202B804440FFFFFF3FFC8C5AC0000000A00380444000000000CF515AC0000000E0887F444000000020C63B5AC000000060457F4440000000E046035AC0000000C069804440000000C098E459C000000000F57F4440000000607AD859C0FFFFFFDF0A80444000000020B7A959C0000000E0C27F444000000000BBA759C0FFFFFF5F07804440000000A0068359C000000000C17F444000000060FA8259C0000000001F5F4440000000A0EA8259C0000000E041594440000000A0048359C0000000C02D374440FFFFFF5F038359C000000000DC2B4440000000C0448359C0000000E0DCFF434000000080228359C000000020CBC8434000000000188359C0FFFFFF1F0AC84340000000E0088359C00000006039904340000000E01A8359C000000060BC844340000000E0038359C000000020A558434000000020048359C000000040C84E4340000000C0E28259C0FFFFFFBFB421434000000020EB8259C0FFFFFFBF7C20434000000000C98259C00000006000DE4240000000E0D08259C0000000601BD24240 Colorado 08 0.55034722222222222222 0.55845410628019323671 0.53461975028376844495 0.52366863905325443787 0.56389776357827476038 0.54117647058823529412 0.61495844875346260388 0.62442396313364055300 0.56059009483667017914 0.63647798742138364780 0.57397107897664071190 0.53067185978578383642 0.55756013745704467354 0.57445442875481386393 0.65222849968612680477 0.66604127579737335835 0.72323600973236009732 0.68924302788844621514 0.76779661016949152542 0.80066079295154185022 0.79224376731301939058 0.73301204819277108434 0.81230212573496155586 0.78431372549019607843 0.74090723334695545566 0.73416149068322981366 0.73961218836565096953 0.71298654056020371044 0.74875444839857651246 0.78982057854265836690 0.79837570621468926554 0.79972658920027341080 0.79454306377383300460 0.77606783919597989950 0.78123076923076923077 0.77565421934725080859 0.78146804353893385431 0.76974703149199793495 0.74898688915375446961 0.76097231600270087779 0.76047039405818031772 0.79666011787819253438 0.83264150943396226415 0.84096893101632438125 0.85082518827111039897 0.86070747101130192279 0.87318690426854537920 0.87444514901712111604 0.86857208448117539027 0.87310838445807770961 0.87466958344727007565 0.86896052737358308546 0.87565813198701767039 0.86861705696839562504 0.85891512121020317742 0.83907849829351535836 0.82161701220487128924 0.78710450144725022457 0.74980589175610870062 0.71999833381930270338 0.71899833313951234640 0.73694643925792938360 0.76264750772437925772 0.74897852278679937140 0.76096209715559759476 0.76961876064456963186 0.77832034306820671738 0.78366993307839388145 0.78520058433531857512 0.79728719427533910072 0.80269720101781170483 0.81075295275590551181 0.80957449324723858180 0.80823873777397015778 0.80409453276097929148 0.78968650714301584128 0.79511717919271830207 0.77566346586353235680 0.76330428333024290747 0.76509316314553990610 0.76025864684466019417 +7 0106000020E610000001000000010300000001000000250000000000006046ED52C0FFFFFF5F5B474340FFFFFFFF81ED52C00000008024534340FFFFFF5F65EE52C0FFFFFFFF426A4340000000E02FF052C0000000401E92434000000000BFF052C000000060B69F434000000040F0F052C0FFFFFF5FDEA54340000000E073F152C00000006006B14340000000E0A6F252C000000000A4DC4340000000009EF152C000000080BADC434000000060BDEF52C0000000402DE34340FFFFFFBF7CEC52C00000002001E94340000000E03CE952C0000000004AEB4340000000E05CE552C00000006085EB4340000000201ADE52C0FFFFFF9FC8E94340000000E0EEDA52C00000006041E643400000002060DA52C00000008013E543400000008064DB52C0FFFFFFBF99E34340000000007DDD52C000000020B2E143400000008068DE52C0000000A0F0DE4340000000407CDE52C00000000028DC43400000004056DF52C0000000C07CDB4340FFFFFF1F16E752C0FFFFFFFF6FCE43400000002008E452C0000000608AC84340000000A0C5E552C0FFFFFFBF5CBB434000000020FFE052C0000000E0F7AE434000000040C2D952C000000040FAA043400000006074D952C00000004059894340FFFFFF5FCAD452C0FFFFFFDF95814340000000A0B4D352C0000000A0137943400000006038CC52C000000080826743400000002052C552C00000004060664340000000A0F1C252C0000000E088394340000000005FC452C00000006098394340FFFFFF3FF5C552C0FFFFFF5FA8394340000000C069D652C000000040443A4340000000E0BEEC52C0000000C0453B43400000006046ED52C0FFFFFF5F5B474340 Delaware 10 0.89583333333333333333 0.82801932367149758454 0.87968217934165720772 0.87278106508875739645 0.90095846645367412141 0.94852941176470588235 0.97091412742382271468 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 0.82862644415917843389 0.91902071563088512241 0.94058786741713570982 0.92822384428223844282 0.89072282299373932840 0.94011299435028248588 0.92786343612334801762 1.00000000000000000000 1.00000000000000000000 0.97467209407507914971 0.93617021276595744681 0.95668165100122599101 0.95486542443064182195 0.99208547685001978631 1.00000000000000000000 0.94128113879003558719 0.97070670084218235079 0.96186440677966101695 0.95864661654135338346 0.92537804076265614727 0.92116834170854271357 0.93815384615384615385 0.94384004704498676860 0.96790399106893664527 0.93185338151781104801 0.90226460071513706794 0.91807337384650011254 0.91211058386630905715 0.90530451866404715128 0.92301886792452830189 0.93084079340003510620 0.94071462906585483096 0.93160135035960663438 0.92954828014919187733 0.93202282815472415980 0.90828741965105601469 0.88527607361963190184 0.87038556193601312551 0.86847817348661467964 0.85632888568337540570 0.85398912970542843723 0.85632002025444648395 0.84277588168373151308 0.85572669615733091723 0.83745882822636989720 0.81904544416533455127 0.80443204065480901404 0.81133465131604450130 0.80924596050269299820 0.83170159699214533001 0.80649554740701938188 0.80477670427673805824 0.80341936329097340495 0.79478511284314646133 0.79588910133843212237 0.76988987526688391954 0.78957065043255366870 0.78315521628498727735 0.76906988188976377953 0.77258213297381678848 0.79633992527545750934 0.80482569932543987924 0.79979559643626830190 0.79318191484836884862 0.76971818590733968279 0.74635638791025403301 0.74636883802816901408 0.76105506674757281553 +19 0106000020E610000003000000010300000001000000590000000000002078D451C000000000D4E2444000000080C7D551C0000000C064E44440000000A024D651C00000006012E844400000006071D551C000000080CCED4440000000A0F3D551C0FFFFFF5F16F044400000008059D551C0000000E0ADF24440000000E09AD851C000000020B5F144400000008081D851C00000004055FD4440000000E043D851C000000020BE01454000000000DEDF51C0000000002C014540FFFFFF9F17F351C0000000E0880045400000008061F351C0000000E049024540000000E01B0652C0000000404A034540000000C0C10852C0000000005E034540000000C0832052C000000000ED034540000000A0962452C000000000D8034540000000A0452552C0000000E0C002454000000040EE2652C000000000E8024540000000200A2752C000000080E503454000000020683052C0000000005204454000000080273152C0000000E043004540000000605C3452C0000000A0A0FF444000000040483452C0FFFFFFDF46044540000000406B4052C0000000E09804454000000000F34252C0000000C0A2044540000000C0045F52C0000000200F06454000000020F45F52C000000040E7094540000000407B5652C00000002099404540000000608B5052C0000000407C5F4540000000204A4152C0000000A0C25E4540000000E0193B52C0000000405F5E454000000000331D52C000000000E65C4540000000E0F11152C0FFFFFF7F355C45400000008090FB51C000000000835A4540000000E0B0F951C000000000475A45400000008069D251C0000000E068594540FFFFFF9F2FD051C0000000E0EC5C4540000000206CCF51C0000000E0295F454000000040F5CB51C0000000C08C5E4540FFFFFF9F9ECB51C0000000405367454000000020C0C751C000000080BA684540000000603AC451C000000000F1664540FFFFFFBFA8C151C000000040F06C454000000060FFBA51C0000000A049714540000000E082B951C00000004082714540000000606AB651C0000000E0816E4540000000E01EB451C000000000F96E4540FFFFFF7F5FAF51C000000040EB54454000000060FFA551C000000000B7524540000000E092A851C0000000E0904A4540000000A012B451C000000080EE4545400000002039B951C000000020573945400000002083BD51C000000080553745400000000038C251C0000000408C244540000000201EBB51C000000060011E4540000000E029B951C00000006001224540FFFFFF9FCFB451C00000000055214540000000609BB151C0FFFFFF1FD01F454000000080F0AB51C0000000A09713454000000060A1A751C000000040EAFB4440000000809DA251C00000000026F744400000006072A251C0FFFFFFBF1FE7444000000080239B51C0FFFFFF7F2BDF4440000000408F9151C0000000E05BDC4440000000C0DD9551C00000002019DB4440000000C02B8D51C00000000032DB4440FFFFFF9F438151C00000004005E4444000000020108051C0FFFFFF5F99ED444000000040778651C00000008044004540000000005D9051C0000000A0AE074540FFFFFFFFAD8851C00000002044094540000000A0438351C0000000605A034540000000C0BD7D51C0FFFFFFDFB5F34440000000C0C57A51C0000000E03EE24440000000201E7D51C0000000E0EFD54440FFFFFF3F7B9951C00000002065CE4440000000A0BD9B51C000000020EAC8444000000080CFA851C0FFFFFFBF14C544400000002096AA51C0000000A02BC74440000000C0B2A751C0000000E025DE4440000000C0C1B551C00000000034D044400000002021B951C0000000A020D14440000000E01BC051C0FFFFFFFF8FC244400000008087C751C00000004019BF44400000000012C951C0000000A0DCD34440000000A0C1CC51C0000000C0D5D64440000000E0AFCE51C00000006092DA4440000000C018D151C000000040F4DF44400000002078D451C000000000D4E244400103000000010000000F00000000000000B6A651C000000080FBB64440000000C05DA451C00000004073BB4440000000406AA351C00000004069B5444000000080E3A451C00000008080B444400000000001A151C00000008003B34440000000A0259F51C000000060B4AB4440000000204FAF51C0000000E0C1AA4440FFFFFF9F4BB151C00000008026A64440000000600BB651C0000000409BAC4440000000801DB251C0000000A01BAD4440000000805FB151C00000004095A94440FFFFFF9F25B051C000000040E7B04440FFFFFFBFC0AA51C0000000C037BA4440000000C0CEA651C000000020BEBC444000000000B6A651C000000080FBB644400103000000010000000D00000000000080108251C000000060E9A7444000000080738051C0000000408EA9444000000000BD8151C0000000C025AB4440000000A0A48551C000000020FBA54440000000E03D8251C0000000A0BFAC444000000000308351C0000000E027B2444000000080DB7D51C0FFFFFFFFBAA54440000000E0037E51C00000008037A0444000000020A28651C0000000E07B9E444000000000AF8D51C00000000092A24440000000E0498D51C0000000A0A0A54440000000604C8651C00000006085A3444000000080108251C000000060E9A74440 Massachusetts 25 0.78645833333333333333 0.80772946859903381643 0.86152099886492622020 0.90680473372781065089 0.89297124600638977636 0.89558823529411764706 0.89058171745152354571 0.82258064516129032258 0.77133825079030558483 0.84528301886792452830 0.80533926585094549499 0.75851996105160662123 0.77405498281786941581 0.68870346598202824134 0.79221594475831763967 0.81238273921200750469 0.81995133819951338200 0.79453614114968696642 0.81073446327683615819 0.83259911894273127753 0.81994459833795013850 0.79807228915662650602 0.82180009045680687472 0.79057154776804338757 0.79566816510012259910 0.79917184265010351967 0.81954887218045112782 0.79810840305565660240 0.81708185053380782918 0.84987184181618454778 0.85805084745762711864 0.85816814764183185236 0.85634451019066403682 0.85866834170854271357 0.86123076923076923077 0.86209938253454866216 0.86547585821936924365 0.85983479607640681466 0.85411203814064362336 0.87846049966239027684 0.86795956261605116567 0.88133595284872298625 0.89584905660377358491 0.89626119010005265929 0.88944079474443198205 0.88419198590929106121 0.88948749827324216052 0.88700063411540900444 0.87649219467401285583 0.86707566462167689162 0.86336705860906024975 0.85802717260229922019 0.85322755138838802741 0.85908877407233442931 0.87720741819102474840 0.88447098976109215017 0.89111549325800778127 0.89599760455135243038 0.89079698561315368806 0.88865747490315324697 0.87056634492382835213 0.86860412926391382406 0.88407847224807355843 0.86872708224201152436 0.86696844807783257888 0.87911044150399580768 0.87804801702820296115 0.88485898661567877629 0.88021125969210023598 0.89164797607604400299 0.90460559796437659033 0.93484251968503937008 0.92828591026277821141 0.93377120963327859880 0.94035095995094108213 0.92852540602990513009 0.92552422270426608821 0.91707899008468112569 0.91527906545521972928 0.92809198943661971831 0.94034435679611650485 +41 0106000020E6100000060000000103000000010000001703000000000020E57F5AC000000040D1643F40FFFFFFBF9D8D5AC0010000A0667A3F40000000C083985AC0FFFFFFDFD7BB3F40FFFFFF7F7EA25AC00000004044C93F400000000053A75AC0000000A056D13F40000000A065A75AC0FFFFFFFF39D83F40000000A02FA95AC00000006025E53F400000006086A85AC0000000C0FBE93F40000000A073A85AC0000000C0DCF83F40000000A091A95AC000000040F0FA3F40FFFFFF7FE0A75AC00000006020004040000000A02E985AC00000002015004040000000602C805AC0000000E032004040000000C0FE3A5AC0000000E08B004040000000A06F365AC000000040670040400000004034015AC0FFFFFF7FEE00404000000000C7FE59C000000000C100404000000080A7EE59C000000000C8004040FFFFFFFF40D559C00000002088004040000000C0B5C359C0FFFFFF3F3E004040FFFFFF3F88C359C000000020E10A4040000000A0CFC359C0000000A0F94140400000002020C359C0000000600D7A4040FFFFFFFFB9C259C00000006059B040400000008072C259C0000000406AC84040000000C018C259C000000040BDE9404000000080DDC159C00000002064274140000000006BC159C0000000E0645F41400000008095C159C0000000E07B7B414000000040A4C159C0000000C0AE964140FFFFFF1F65C159C000000040D2CF4140FFFFFF5F6AC159C00000004003DF4140FFFFFFFF81C159C0000000E02B07424000000020B7C159C0000000A0EB3E424000000080CDBF59C000000020053F424000000000938A59C000000020BF3E424000000080308259C000000020193F4240FFFFFF3FB36759C000000000FA3E424000000020BD4559C000000080773E4240000000003E3D59C000000080AB3E4240FFFFFF9F292359C000000040A63E424000000080700059C000000060373F4240FFFFFFBF120059C0FFFFFFDF0A3F424000000060D1FF58C0000000C05D074240000000C0DAFF58C00000002020F1414000000000000059C00000002035CF414000000080A3FF58C0FFFFFF5F58B64140FFFFFFDFD1FF58C0000000C051974140000000A0BFFF58C0000000C0F7834140000000A0EDFF58C0000000C0A35F414000000000C0FF58C000000020FA474140FFFFFFDF36FE58C000000020EB474140000000A076FC58C0FFFFFF5F2F4A414000000060A4FB58C000000040204A4140000000C05BF858C0000000A02A464140000000A013F758C00000006062424140FFFFFF9F1DF558C0000000403A404140000000A0C5F158C0FFFFFFBFD438414000000080D5EB58C00000002050304140FFFFFF1F7EE658C0000000E02C2F41400000004074E558C0000000004331414000000080FBE458C000000040573441400000008072E358C0000000A0243541400000000023E058C0FFFFFF7FB833414000000020AFDE58C00000004017314140000000600EDC58C0000000A0AE2E4140000000C03CDA58C0000000E03E2F4140000000E039D958C000000080C83241400000008023D958C000000040E9364140000000204FD758C000000000A0394140000000E0B0D458C0000000A0D33441400000006019D158C000000000FB3241400000004043D058C0000000A0212F4140000000C026CD58C0000000C07E2A4140FFFFFF3F90CC58C0FFFFFF3F0E2741400000002018CD58C000000000B9204140000000E030CC58C000000060A31C41400000002046CB58C0000000C03A1B4140FFFFFF3F30C858C0000000C0C91941400000000005C558C000000080AB1A41400000000041C258C00000004076194140000000A0C1BF58C0000000C0D01A4140000000E0F5BC58C000000080E7184140000000C00BB958C0000000C095144140FFFFFF7FE8B358C000000000AE124140FFFFFF7FD3B158C000000020E41041400000008023AD58C000000040BB10414000000060A9AB58C00000002033134140000000A059AA58C0FFFFFF1FD21241400000004010A858C0000000A04714414000000020DDA658C00000000061134140000000A0E2A458C0000000C02A12414000000060AFA358C0000000A07B0D414000000020F89F58C0FFFFFF3F8008414000000020AF9C58C0FFFFFFBFF506414000000040F79A58C0000000206D084140000000800E9A58C0FFFFFFDF8D0A414000000060059958C0000000602A0B4140000000A0979858C0000000E0D10E4140000000006D9658C00000000031124140000000E0829458C000000080D811414000000060BA9158C000000040BA0F4140000000E00F8B58C000000060C40E414000000060C28858C000000020B8114140000000005A8758C00000000012134140FFFFFF1F068658C0000000203911414000000080158758C0000000E0EF084140FFFFFF5F848558C000000020AE004140000000408E8358C0000000C0B1FE4040000000E0808158C0FFFFFF7F55FE404000000040E47E58C0000000202A00414000000080D07C58C0000000004FFC404000000000A87C58C000000020D9FA4040000000C0A17D58C0000000606DF9404000000000D87C58C0000000C05CF74040000000E0787E58C0FFFFFF1FBEF44040000000007D7E58C00000004085F34040000000601A7D58C0000000E015F14040000000202E7A58C000000000E0EF404000000000AA7758C00000006074ED404000000020907658C000000080B4ED4040000000C0927258C000000080FAF1404000000040687058C0000000004FF7404000000040A86E58C0000000C03AF84040000000A0126D58C0FFFFFF9F5BFC4040000000C0F26A58C0000000E08AFE4040FFFFFF5F696658C0FFFFFF7F16FC404000000020E96558C0000000407DF54040000000C0D76458C0FFFFFF1F86F34040FFFFFF5F7E6358C000000020B3F34040FFFFFF3F2A6158C0000000C058F54040000000E08F5E58C0000000C02FF44040000000E09D5D58C00000004081F3404000000080405D58C0000000A0F9F14040000000A0F95C58C00000000009EB4040000000603F5A58C0000000000DE94040000000A0405758C0000000005FEA404000000020E05558C00000004053EE404000000040285458C00000000069EF4040000000001A5458C0000000E0AAF24040000000006D5158C000000080B0EF4040000000E0E35058C0000000E0EAED4040000000400B5058C000000080BDEF404000000080BF4F58C00000006076F2404000000080864D58C0000000A0EDF34040FFFFFF5F044C58C00000002019F34040000000C0814A58C0000000A07BEE4040FFFFFF3FCA4A58C00000008084EC4040000000207B4C58C00000004007EB404000000020554D58C000000040EAE84040000000401B4C58C0000000C05AE0404000000020C24958C00000000045DD404000000060654758C000000060EBDC4040000000C0CA4558C000000060A7DD404000000080574558C0FFFFFF5F07DF4040FFFFFF5F9C4558C0000000805EE74040000000A0334358C0000000C066E94040FFFFFFFF014558C0000000603DEB404000000060424558C0000000E0F0EC4040000000A0894458C000000040A9ED404000000060A34158C00000008097EB4040000000E05F4058C0FFFFFF9FDDEC4040000000A0363F58C0000000A02EF0404000000020393F58C0000000A0DBF84040FFFFFFBFF63D58C000000020FAF74040000000C0EA3B58C00000002053F94040FFFFFFFF7D3B58C0000000601BFB4040FFFFFF3F803958C0000000809AF94040FFFFFF9F803858C00000000059F64040FFFFFF7F403858C00000000027F14040000000E01A3758C0000000804BEE404000000040043658C000000000D4ED4040000000801A3458C00000002096EF4040000000C00B3358C0000000805AEF4040000000C0EC2F58C00000006076EA4040000000208C2D58C0FFFFFF3FBCEA404000000060602C58C00000002088EC4040000000805F2B58C0000000E0C0F34040000000A0A32A58C000000000EFF4404000000040682558C0000000E0B4F24040000000804E2758C00000008073EE4040FFFFFFFF792658C000000000E6EB404000000000FA2358C000000060A7E9404000000040AD2058C00000006068E84040000000400C2058C000000020E0E4404000000020311F58C0000000C099E34040FFFFFF7FD81A58C0000000E0E7E4404000000080BB1758C000000040C5DE4040000000E03E1658C0000000C04EDA4040000000E03D1458C0000000A0D4D9404000000020401358C00000000066DB4040000000208A1258C00000000087E14040FFFFFFFFCB1158C000000060FEE24040000000609A0D58C000000040DBE0404000000040F80B58C00000004019E14040000000E0CD0A58C0000000407AE2404000000000530A58C0000000602CE64040000000000D0958C00000004000E94040000000A0E30958C00000000077E9404000000000910B58C0000000C07AE7404000000060B80B58C0000000E06BE8404000000040D40A58C0000000201CEA4040000000A0880958C0000000A0F4EA404000000020010758C00000002025EA4040FFFFFF7FDB0558C0000000201BEC4040FFFFFFFF110358C000000000AFEB404000000040B60158C0FFFFFF1F92ED404000000080E60058C0000000000FEC4040000000601D0058C0FFFFFF7FB1ED4040FFFFFFDF2A0058C000000040CBEF404000000020A1FF57C0000000600CF04040000000A08DFE57C000000060D1ED4040FFFFFF5F5CFD57C0000000A0B9EE4040000000405BFC57C0000000A0EAF14040FFFFFF7FB7FB57C0000000E0FCF14040000000002EF657C000000020A7EB4040000000C0DCF457C0FFFFFF3FE8EB404000000020E9F257C0000000A0ADEE4040000000602FF157C0000000C0FAEC4040FFFFFF7FE9F057C00000004083F0404000000040AFF057C0000000405CF24040000000A0CCEF57C000000080A2F3404000000000C8EC57C0000000A089F24040000000208BE857C000000000C6F54040000000203BE757C0FFFFFF5FCAF540400000006059E757C000000080E5F74040000000E0C9E657C0FFFFFF1FE7F840400000008004E457C000000040D1F74040000000E0F6E257C000000060B7F34040000000C040E157C0000000E00CF4404000000000B6E157C0000000A0EBF24040000000200AE357C00000000053F2404000000080D1E257C00000000060F1404000000020D3E057C000000000E9F2404000000040EDDF57C000000020DCF04040000000C0F5DD57C0000000A076F1404000000020E7DC57C000000000D1EE4040FFFFFF5F1FD557C0000000407AEF4040000000C084D557C0000000A0D4F240400000004053D357C0000000E07CF14040000000E054D257C00000000086F1404000000020C0D157C0000000007FF54040FFFFFF1FDFD057C000000020EBF240400000004010D057C0000000C0D7F34040FFFFFF1F15D057C000000060DDF7404000000080FACE57C0000000A080FB4040000000007EC957C000000020C6F84040000000A030C857C0000000606EF84040000000801BC857C00000000065F5404000000060A1C757C0000000A0C5F44040000000601AC657C000000080FBF54040000000E043C557C0FFFFFFDF8FF54040000000E0BDC557C000000020CEF24040000000C059C557C000000020B9F140400000000010C457C00000008075F54040000000800AC457C0000000E0C6F2404000000040BEC257C0FFFFFF7F35F140400000002064C257C0000000E0E7EE404000000040D1C057C0000000605AEF40400000006050BF57C00000006097ED404000000040FFBD57C000000020E0EE4040FFFFFF1F6FBD57C0000000C08DEC40400000002027BC57C000000020A0EB4040000000802FBC57C0000000606CE8404000000060C4BA57C00000002079E84040FFFFFF9F25BA57C000000060D8E64040000000007DBA57C00000008011E54040000000C06CB857C00000000032E3404000000080E7B657C0000000C0E9DF4040000000206DB457C0FFFFFF7FECDF4040FFFFFFFF67B357C0000000A0AADE40400000000025B257C0000000E06AE0404000000040E8B057C0000000205DE04040FFFFFFBF0CB257C0000000A002DF4040000000401FB257C0FFFFFFBFE8DD404000000040FCAF57C0000000604CDE404000000060D0B057C0FFFFFFFFBFDB4040FFFFFFBF7EAF57C0000000C009DC40400000006049B057C00000004098DA40400000004077AF57C000000020C3D940400000002039AC57C0000000605BD8404000000000C8AA57C00000002028D9404000000060F3A957C0000000009DD84040000000A03CA957C000000040BDD64040000000C0BFAA57C000000060F2D54040000000E0D7AA57C00000008041D540400000008025AA57C000000060F5D4404000000080E1A857C0FFFFFFFFC5D54040000000606EA857C0000000008AD74040000000E075A657C0FFFFFF9F32D54040FFFFFF5F72A557C0000000E0E8D640400000004006A557C000000000D2D54040000000E0E2A357C00000004001D64040000000602CA457C0000000A0DDD440400000004073A557C0FFFFFFBFC0D4404000000020A8A557C0000000C0E5D34040000000C0E4A457C0000000E079D3404000000020E8A257C000000000B0D4404000000000AFA257C0000000C0F9D2404000000000FBA357C00000004048D2404000000040FAA357C00000004059D140400000006036A357C000000020FCD04040000000C026A157C0000000204ED24040000000809AA157C0000000A07DCF404000000000ADA057C000000060BED04040000000000AA057C0FFFFFFFFBFCF4040FFFFFFBF7E9E57C0FFFFFF3FE4D0404000000000E69B57C00000000077D14040000000E0EC9B57C0000000C0F4CE404000000040E69C57C0000000405BCD4040FFFFFF7F5F9C57C0000000405ACC4040000000006C9B57C0FFFFFF1F6FCC404000000040059A57C00000000068C94040000000C02D9957C00000004098C9404000000060439857C000000020F2CB404000000060B89757C00000008086CB4040000000E0D39757C0000000004DC94040000000004C9957C000000000B8C7404000000080BA9757C0000000801AC64040FFFFFF3F0A9557C0000000805CC94040000000405A9357C0000000A049C74040FFFFFFDF1F9357C0000000A038CA4040FFFFFFDFDA9157C0000000406FCB4040000000C0699157C000000060D4CA404000000020929157C000000000E7C74040000000E02E8F57C000000080D4CB4040FFFFFF3F468E57C0000000E0F8CA4040FFFFFF3F108F57C000000060E0C74040FFFFFF1F7F8D57C0000000206CC7404000000060248D57C0000000E0E3CA404000000080358A57C0000000C000CC404000000040EE8957C00000004096C8404000000020518657C00000000058C94040000000C08B8557C000000000BFCA404000000080EE8357C000000020E2C94040FFFFFF9F4C8257C00000002028C7404000000020568257C0000000009AA24040000000E07A8257C000000020FB824040000000A0A98257C0000000C0F0704040000000A0928257C0000000A0EF58404000000040418257C0FFFFFF1FD2314040FFFFFF9F398257C00000008087194040000000803E8257C00100006098FE3F4000000000A28057C0010000E037FD3F4000000000488057C0000000605AFA3F40000000A08A7E57C0FFFFFF7F37F23F4000000040147E57C00000008054EC3F4000000000E37B57C001000020D2E83F4000000040BF7A57C0FFFFFF3FE2E83F40000000001A7B57C0FFFFFFFF80E43F40000000808D7957C0FFFFFFFFFAE43F40000000201F7957C0000000A0BCDE3F40FFFFFF9F667857C00000006015DF3F4000000060277857C000000000A1D93F4000000040597757C0000000C038D13F40000000A0657557C00100000051CD3F40000000C09C7457C0000000A04EC63F40000000C0317557C000000000D7C03F40000000E0D67357C0FFFFFF5FF8BA3F40FFFFFF1F287457C0010000A05CB63F40000000E0C27357C00000000023B53F4000000080B47257C0000000001EB63F4000000040F57357C000000060B0AC3F40000000809C7357C0FFFFFF3F5DA73F4000000080247457C000000020E1A53F4000000020747457C0FFFFFF3F3B9E3F40000000207A7557C0FFFFFFFF7C9D3F40000000A0497557C00100004016973F40000000A03E7457C000000080BD933F40FFFFFF7FDF7357C0FFFFFFBF1E8F3F40000000A0ED7157C0000000E0A2883F4000000000DA7057C0FFFFFF7FDD873F40000000C0D76F57C0000000E0A7893F40FFFFFF7FD36E57C0000000C099853F40FFFFFFBF2B6D57C0FFFFFFFF43853F4000000000046E57C0FFFFFFBFD27E3F4000000020077057C0FFFFFF1F957D3F4000000060147057C0000000C0497C3F40000000A0836E57C0FFFFFF1FA0753F40000000E0B26C57C00000002022763F4000000060EC6C57C0000000803D723F40000000E0F76B57C0FFFFFF5F29703F40000000608D6C57C001000020806D3F40000000C0716C57C0000000E0796A3F40000000E0FF6B57C000000020F8673F40000000407F6A57C0000000E0F8653F40000000004F6A57C000000040555F3F4000000080A16857C0FFFFFF1FB35F3F40000000A0546B57C00000002011543F40FFFFFF1F9F6B57C0000000C00B503F4000000000FE6957C00000004063493F40FFFFFF5F516957C0000000A04E4A3F40FFFFFF7F5F6857C0000000801E463F40FFFFFFFF736757C0000000209B463F4000000000296757C0000000E020453F40000000A01A6757C000000000003E3F4000000080CB6557C0000000C0CC3A3F40FFFFFF3F966657C000000000F6323F4000000020036657C000000080212E3F40000000A0EC6457C0FFFFFF5F112C3F40000000E03C6357C0FFFFFFBFE0303F40000000E0D96157C0000000E08E2F3F4000000040B96157C001000060962D3F40FFFFFF7F5E6257C0000000A0242D3F4000000020D06157C0000000A0B6293F4000000000D46257C0FFFFFF1FBF283F4000000080666257C0FFFFFF9FE7213F4000000040CC6157C0000000A03C203F40000000E03E6257C0000000E0B61D3F40000000A0A06357C0FFFFFFDFFD1B3F40000000C0D66357C0FFFFFFBFBC193F4000000080C26257C0010000A041183F40000000A0D26257C0FFFFFF5F16153F4000000080166157C0000000A01D133F40FFFFFFBFA56157C000000040960E3F4000000040766057C0FFFFFFDFF5093F4000000000046357C0000000C09E033F4000000000286457C0FFFFFFBF9F043F40FFFFFF5F586457C0000000004F033F40000000C0886457C0FFFFFF1F4DFF3E40000000A0E66357C0FFFFFF5FDFFD3E4000000020A36457C0010000C0E6F93E40000000801D6357C0010000605EF83E40FFFFFFBF636257C0FFFFFF5FF6F43E40000000600F6257C0FFFFFF7FF2F53E40000000C0A36157C0FFFFFFDF91EF3E4000000040EB6157C0FFFFFFFF4DED3E40000000002D6357C0FFFFFF3FC5EC3E4000000020FA6257C0FFFFFFFFC3E73E4000000060206457C000000000E5E63E40FFFFFF3F626457C000000040E0E23E40000000E0E46357C00000008033DF3E4000000020616357C0000000803BDC3E40000000A0406457C0FFFFFF9F5BD83E40000000A08F6357C0FFFFFFBFA3D73E40000000603E6357C0000000600ED43E40000000603D6557C00000008052CD3E4000000080736557C0FFFFFFDFADC53E40000000C0946757C0FFFFFFFFEBBE3E40000000C0E36657C00000000065BB3E40FFFFFFDF896757C00000004088BB3E40FFFFFFBF316757C000000020D8B53E40000000E0866757C000000040D2AF3E40000000403D6A57C00000008040AC3E40000000E0636B57C0FFFFFFDFC2A33E4000000020586C57C0FFFFFFBFD9A33E4000000040D06B57C0000000C0989F3E4000000020556C57C0000000C0A49D3E4000000040FB6A57C0000000800B993E4000000000616C57C0010000404D993E40000000A0F06D57C0FFFFFF7F5E963E40000000C0F16D57C0FFFFFF7F72913E40000000400F6F57C0FFFFFFFFA68B3E4000000040266D57C000000000DA853E40FFFFFF9FBC6D57C0FFFFFF1F4F813E4000000000446D57C0FFFFFF9F097F3E40FFFFFF1FC06D57C0FFFFFFBF167D3E40000000A0AB6C57C0FFFFFFFF54783E40000000E0046D57C00000004067763E40000000A0946C57C00000006050713E40000000A02D6E57C0000000C0D76E3E4000000020866F57C0000000A0A8683E4000000000517057C0000000E0BC613E4000000080DC6F57C0000000A00E5E3E4000000000997057C001000040A95A3E40FFFFFF5F967057C00000006043573E40000000A0B46E57C0FFFFFFFF0E4E3E40FFFFFFDFBF6C57C0000000A0214C3E4000000060456D57C000000080473D3E4000000000C06D57C00000000066383E4000000040146D57C0000000E04C2E3E40000000008E6C57C000000020F92C3E40FFFFFF3FC76C57C0FFFFFF5F9B263E40000000A0B86B57C0FFFFFF7FF2253E40000000C0E66B57C0FFFFFF1F29243E4000000080B66C57C00000006027243E40000000609A6C57C0FFFFFF9F301E3E4000000020566D57C0000000A05F1D3E40000000A0D06D57C0000000C07D183E4000000040996D57C0000000407E0F3E4000000020A77057C0FFFFFFDF86013E40000000A0DD7657C0000000809BFD3D4000000020CE7657C0FFFFFF1FF0F63D40000000C0E97C57C00100004080D13D40FFFFFFFF6F7557C0000000A0B0AC3D40FFFFFF9F2F8457C00000004090AC3D4000000000D99657C0FFFFFF9F558F3D40000000E0209857C0FFFFFFDF4D8D3D4000000060AEAB57C0FFFFFFDFD26E3D40000000200FB157C0000000A02E5D3D400000008041B257C0000000601D623D4000000080A4AB57C0FFFFFFDFA0793D4000000000A7A457C00000002076883D400000000015A057C0FFFFFF5F7C843D40FFFFFF1F119E57C0FFFFFF1F898E3D4000000020B1A057C0000000C08E8B3D40FFFFFF1F28A257C0FFFFFFDFD18D3D40000000C01FA457C00100004039943D400000000070B257C0000000E0DE893D400000000036AD57C0FFFFFF7F94A83D4000000060D1AC57C0000000602BC13D400000002016AF57C00000002001CB3D400000002015B557C00000000086C23D4000000040C7B857C0FFFFFF5F25AB3D4000000080AFBB57C000000040A5AE3D4000000020A6C557C0000000A0D1CD3D40000000E095C257C0FFFFFFFF29B63D400000004051BF57C0000000E000AE3D4000000060E7C057C0000000002C8F3D400000006050BA57C0000000E015803D4000000060E6BE57C000000020E5753D400000008066BC57C000000060F5763D40FFFFFFDFF5BC57C0FFFFFF9F9A6C3D40000000E075BA57C0000000808C6B3D4000000000B0BA57C0FFFFFF7FA4723D400000006008B957C0000000203A663D40000000C02EB457C000000060F55E3D40000000E00DB957C0FFFFFF1FD2643D40000000C085B957C0FFFFFFDF0B4F3D4000000060DFBC57C0010000A06F533D40000000603FC457C00000000025323D400000000046CA57C00000004035333D40000000C08BCA57C0000000A0171E3D4000000040A1CC57C0000000E0EF1A3D40FFFFFFDFE5CF57C0010000E077FA3C4000000080B3E157C0FFFFFF3FA1CD3C40000000C0B6EB57C0000000A019BA3C40000000E0F6EA57C0000000C0AFC03C40000000A053F257C0000000A026BD3C40000000E0FCFB57C0FFFFFF9FC1B03C400000008031FD57C0FFFFFF7F679F3C4000000000F0EC57C0000000800FB83C40000000A0380D58C0FFFFFFDF067D3C400000002077FF57C000000040AF983C40000000C0F5FE57C0000000C033A73C40000000A0340F58C00000002042923C40000000404C0F58C0FFFFFF9FDC983C4000000000140A58C0FFFFFF9F799C3C40000000A0630F58C00000002086A23C40FFFFFFFFAA0958C0FFFFFF7F3EC33C4000000040940D58C0010000E0CCAF3C40FFFFFF5F4D1258C0FFFFFFBF66A93C40000000E04D1158C0000000C07FB53C40000000C0DF1458C0000000A053A23C40000000604E1758C000000000349E3C40000000E0121958C0FFFFFF9F95AB3C4000000080221958C0FFFFFFFFDCB93C4000000060551B58C00000008046B63C4000000080C71C58C0000000004AC13C4000000020AA1B58C0000000E07EB23C4000000040D11958C0000000C030B83C4000000060CD1A58C0000000807FA33C40FFFFFF7F061858C0000000C02E9C3C40000000E06F1F58C0FFFFFFDF938E3C4000000060FA1B58C000000060D4983C40FFFFFF9F141D58C000000040EBA73C40000000E0ED1E58C0000000201A993C40000000E0C22058C0FFFFFFBFB19B3C4000000040C02058C0FFFFFF5F48A63C4000000060812458C0FFFFFF5FE2A23C40FFFFFFFF832458C0000000801CB13C40FFFFFF1F9F2458C000000080E4CE3C4000000020E52458C000000000D1B03C4000000020DB2558C0010000E0A4B73C4000000080602958C000000000D2B63C40000000A03D2A58C0000000E0D7AD3C40FFFFFF3FD42658C000000080A69F3C40000000E00F2758C0000000C0168F3C40FFFFFFDF442458C0FFFFFF1FF8923C4000000020241F58C0000000C097813C40FFFFFF5F0B2458C00000008039783C40000000202F2158C0000000C0F8753C40000000007F1E58C000000040DC7F3C40000000A0011958C0000000801E6F3C40000000E0522A58C000000040674E3C4000000080F32C58C00000002017573C40000000400B2D58C0000000C058653C40000000C0682F58C00000000049673C40000000C05F3258C0000000E03D7A3C4000000060BA3458C0000000A01B733C4000000020743258C0000000C03D723C4000000020953058C00000008031693C40000000809F3158C0FFFFFFDF41643C40000000A09F3658C0FFFFFFDFAD673C4000000060723258C0000000803B5A3C4000000040523258C00000008017503C4000000000C63258C0FFFFFF9F78453C40000000A0C93158C0FFFFFF9FB63A3C40000000A06F3358C0FFFFFF5F21363C40000000A0DB3C58C000000060461D3C40000000006A3A58C000000080BD413C40000000606B3E58C0000000C0F3353C40000000803A3C58C000000040D02F3C4000000020683E58C0FFFFFF9F731D3C40000000C0264258C0000000802C233C4000000020824158C0000000E025333C4000000000704858C0000000A063213C40000000A0AA4858C0000000606C293C40FFFFFF5FC04A58C000000060D2283C40000000400D4A58C0FFFFFF1FCB1D3C4000000080A85058C0FFFFFFBF91103C4000000060704F58C000000080740C3C40000000804C5158C000000080A3063C40000000201E4F58C0000000805F0A3C4000000080E04758C0FFFFFF5FE40D3C40000000A0B04158C000000080951B3C4000000000864158C0FFFFFF3F2E053C4000000000564758C0000000C056EA3B4000000080824C58C0000000A0EDCF3B40FFFFFF3FCF4F58C0FFFFFF7F83D23B4000000060A74D58C0000000A0C3D43B40000000A0245258C00000006003DF3B40000000601B5758C00000004007D73B40000000A01E5658C0000000A088DF3B40000000C0AD5E58C0000000C05BDA3B40000000A0C95F58C0000000C01EE03B4000000080636158C0FFFFFF9F16DD3B4000000060F85F58C0000000C0DED73B4000000040B55E58C000000000FED13B40FFFFFFDFDD5858C0FFFFFF5FD8D43B4000000040615958C0FFFFFFBF55C53B40FFFFFFBF565458C00000006054B63B40FFFFFF5F5E5658C0000000C01FB73B40000000207B5458C000000060CDB03B40FFFFFF7F9D5658C0000000800BA43B40FFFFFFBF8C5958C0FFFFFF7F18A23B40000000803D5658C000000000A6A13B4000000020CA5358C00000008036B53B40000000A0FC4F58C00000004057B03B40000000A0365558C000000040F48F3B4000000080625A58C0FFFFFF9F2E523B4000000020076058C0000000C0D5513B40000000807B6058C00000006070703B40FFFFFFFFD06158C0FFFFFFFF16583B4000000040686658C0000000A0D54C3B4000000040017058C0000000406F6B3B40FFFFFF3F856B58C0000000005C4B3B4000000040397258C000000000A8493B4000000000156358C0000000E0EE3A3B4000000080575B58C0000000C0DF433B4000000060396058C0FFFFFFDFDF143B40000000E0A75E58C0000000201BFF3A4000000060636458C0000000E054FA3A40FFFFFF1FB76358C0000000E096D83A4000000080B75F58C00000004035CB3A40000000A0E85C58C000000020DA993A4000000040415B58C000000060AA843A40000000A0615E58C0000000E00F7A3A40000000C0F45A58C00000004093623A40000000C0985758C000000060EB5B3A40FFFFFF7F9D5658C000000000B52E3A4000000020335058C0FFFFFF1F7D113A4000000040AF5158C00000002095003A4000000060A34D58C00000004052023A40000000C0054B58C0000000A05EF43940FFFFFF3FA85358C00000006012F73940000000E07B5358C0000000404CF03940FFFFFF1F625858C0000000E0C1EA394000000060AE5858C0000000A069D83940FFFFFF5FCC5B58C0000000E05ED8394000000000C46558C000000040E8EE3940000000C0CB6458C0000000A044F43940000000203A6758C0000000C045F63940FFFFFF5F786958C00000008000063A4000000000847758C000000060650F3A4000000080908258C000000080340F3A40000000E0E28458C000000040DD083A40FFFFFF5F538558C000000080D5103A4000000020D88C58C0000000202D0E3A4000000040AF9258C0000000601D193A40000000E05D9158C000000000F31E3A40FFFFFF9FB49258C0000000C0FF213A40000000E0FC9458C0000000E0941C3A4000000060389658C0000000409F283A40000000009C9858C0000000A0F1273A40FFFFFF5F049D58C0000000A08D383A40000000E0439F58C00000006098333A40000000E065A658C000000020AD423A400000000063AB58C000000060F73D3A400000002078B458C0FFFFFF9F04603A40000000602BBA58C0000000803E5C3A40000000001DBC58C00000000033653A40000000A0D4C658C0FFFFFF5F666B3A40000000807EC658C0000000E0037D3A40FFFFFF9FCBCA58C0000000E0B48B3A40FFFFFFBF9CCA58C0000000A073943A400000000046D258C0000000007CDB3A40FFFFFF3FFED858C00000006056F23A400000004022D958C000000060DCFE3A40000000C01FDD58C00000008055073B4000000060FADB58C0000000A0FE323B4000000000C7DD58C0FFFFFF1F17453B40FFFFFF1FCAE258C00000004093513B400000004064DF58C0FFFFFF1FA27D3B40FFFFFF1FB6E158C0FFFFFFBF18813B40000000E025E358C000000020D59C3B4000000040BAED58C0000000E05BA93B40000000E034F458C000000020B5C73B40FFFFFF9FFBF758C00000002035CC3B400000006047FC58C000000040A4FC3B400000006092FF58C0FFFFFFBFE2003C4000000000340659C0FFFFFFFF7E273C4000000060B30D59C000000000B2333C40000000404D0E59C000000020D03D3C4000000020111359C000000040C5473C40FFFFFFBFBE1259C0FFFFFF1F03523C40FFFFFF1F801659C000000020E9643C40000000001D1859C0000000E0887A3C40FFFFFF9F211659C00000002035803C40000000A0D91A59C000000020508B3C40000000A0CD1959C0000000C0F8963C40000000C0DD1F59C00000008036A93C4000000020BF2559C0000000C0EBE43C40000000206C2959C0000000201FEC3C40FFFFFF1FCD2A59C0FFFFFF9F7F143D40FFFFFFDF303159C0FFFFFF5FA42A3D40FFFFFFDF013359C0FFFFFF9F143E3D4000000060944059C0000000A08D5F3D40000000A04F4459C0000000C03A793D4000000040BB5059C000000000C7863D40000000204B5059C0FFFFFFBFF5A03D4000000080C55359C000000080B6943D4000000040935359C0000000C005A73D40000000E0935759C0FFFFFFBF3BA83D4000000060A15A59C0FFFFFFBFD4BE3D4000000080AE5959C00000008018C53D4000000000B35C59C0000000C0B5C23D40000000201C5E59C0000000A0E7C93D4000000040746259C0FFFFFF1F55C33D4000000020D06259C00000000064CF3D4000000020376559C0FFFFFFDFE0C33D4000000060F06859C0FFFFFF1FC8C13D4000000000957059C0FFFFFFBF83C93D40FFFFFF7F887359C001000000AEC73D40000000206C7459C0FFFFFF7F6AD03D4000000080267B59C0FFFFFF3FDBC93D40FFFFFFDF4A7E59C0000000209BD13D4000000080188459C0000000A0D9C83D40000000E0C19459C0000000404FE13D4000000020869759C0FFFFFFDF64D83D4000000080A09859C0FFFFFF1F98C43D40000000C032A059C0000000A013C93D400000002053A359C0FFFFFF3FDFBF3D4000000060E5A459C0FFFFFF3F3BC73D40FFFFFF9FCEA859C0000000807ABB3D40FFFFFF7F49AB59C00000008085BE3D40000000A080B359C0000000A0B7873D40FFFFFFFF9EB459C0000000A06E693D40FFFFFF3F83B859C000000080765A3D400000000022BA59C000000080EA443D40000000606FB759C000000020A23A3D40FFFFFFFF3CBF59C0FFFFFF5FDC303D4000000060D2C959C0000000E08AFA3C40000000C00FD159C000000080E8013D4000000040F1D159C00000000083FC3C400000002079D559C001000000E30C3D400000006007D859C00000004038083D400000004057DE59C0FFFFFF5F77123D40FFFFFFDFADE159C0FFFFFF9F8A253D40000000A019EE59C0FFFFFF3FCD303D40000000C059EF59C000000020F83A3D40FFFFFFDF0EF259C0010000E0D33A3D400000000023F159C000000060FF473D40000000205EF259C0FFFFFF1F6B443D40000000A0EB025AC0FFFFFF9FFF533D4000000040850A5AC00000004095663D40000000601A0D5AC0FFFFFF1FEA7B3D40000000802A185AC0000000E0F48C3D400000008041225AC000000080F1AD3D40000000C0F6245AC0FFFFFFDFD4CE3D40000000E0282B5AC0010000C0C6E83D4000000060932C5AC000000060AB0E3E40000000402F2B5AC0FFFFFF7F22263E40000000A0F72C5AC0FFFFFF9F0D3D3E40000000E017345AC000000060B8593E40FFFFFF3F9D335AC0000000E05E603E400000008097365AC0000000606B643E40000000E000395AC00000000010923E40000000E0293F5AC0000000E02DA43E40FFFFFFBFD73F5AC0FFFFFF7F30AF3E4000000040E0435AC00000004018B03E40000000C0B74D5AC0FFFFFFDFE4CF3E400000002086505AC0FFFFFFFF32CC3E400000000068525AC0000000A0FAD43E400000000015545AC0FFFFFF9F06D13E40FFFFFFDFFA585AC0FFFFFF7F63DA3E40000000202E5A5AC0FFFFFFDF0AE73E40000000007B635AC0000000A08FFF3E40000000209B665AC0FFFFFF1F20163F400000004043715AC0FFFFFF3FB82B3F4000000020E57F5AC000000040D1643F4001030000000100000005000000FFFFFF9F75BA57C0FFFFFF3FF0413D40000000C01CB157C0FFFFFFDFAA573D40000000E0E5AF57C0FFFFFF1FCA513D4000000020BFC657C0FFFFFF3FD2183D40FFFFFF9F75BA57C0FFFFFF3FF0413D4001030000000100000008000000FFFFFFDF761958C00000006089583C40000000806A3558C000000080FA103C40FFFFFF1F723358C0000000C0122C3C40000000404A2F58C0FFFFFF1FFC2E3C40000000800E2258C00000008078513C40000000A0A21D58C0000000C069533C40000000200B1B58C00000006035643C40FFFFFFDF761958C00000006089583C400103000000010000000900000000000060283C58C000000040C20B3C40000000C0D53758C0FFFFFFBFA3213C4000000060983558C000000040121A3C40000000C0A43658C0000000A0A50C3C40FFFFFFBF2C4358C0000000C048D73B4000000040904158C0000000E014EA3B40000000C0C23C58C0000000E009FC3B40000000A0453E58C0000000E037003C4000000060283C58C000000040C20B3C400103000000010000000D000000000000C0FB5658C0000000C0A1483B40000000C0475858C000000040E0353B40000000E0115858C000000060DE483B40FFFFFFDF785558C0FFFFFF7FD9703B4000000040EA4F58C000000020C5943B40000000408E5058C000000000D9A63B40FFFFFF7F074D58C000000040B09C3B4000000040E44A58C0000000201CB53B40000000A0D14458C0000000A0AFCF3B40FFFFFFFF3B4758C0FFFFFF1FB8D13B40FFFFFF9F6D4358C0000000E099D43B4000000060514E58C000000020F2923B40000000C0FB5658C0000000C0A1483B4001030000000100000015000000000000C0455358C0000000A0DC993A40000000E0E95658C0000000C0E6B43A40FFFFFFBF675858C00000002004D23A40000000C04C5958C0FFFFFF5F07EC3A4000000080A85958C000000060791C3B40000000E0E15858C0000000609F333B40000000E0395858C0000000C056343B4000000060C35858C000000000E5183B40FFFFFFBF675858C000000020F3F23A40000000E0E95658C0000000607BCD3A4000000000EF5258C0FFFFFF7FC5993A40000000E0DA4E58C0FFFFFF5F0B6B3A40000000C07A4C58C0000000A05D423A40000000C0FE4A58C000000080E5133A40000000407A4B58C00000006069123A40000000E05E4D58C0FFFFFF7F23403A40000000E07B4E58C0000000A03A593A4000000080145058C0FFFFFF7F576B3A40FFFFFF9F1A5158C000000080997A3A40000000C0D35158C0000000003B8A3A40000000C0455358C0000000A0DC993A40 Texas 48 0.41579861111111111111 0.39806763285024154589 0.39500567536889897843 0.39349112426035502959 0.41054313099041533546 0.43235294117647058824 0.45152354570637119114 0.42857142857142857143 0.44046364594309799789 0.50817610062893081761 0.46384872080088987764 0.42648490749756572541 0.45360824742268041237 0.46148908857509627728 0.59259259259259259259 0.65353345841150719199 0.64355231143552311436 0.59590210586226522482 0.64802259887005649718 0.66850220264317180617 0.72022160664819944598 0.65686746987951807229 0.67299864314789687924 0.65248226950354609929 0.65304454434000817327 0.67494824016563146998 0.67154728927582113178 0.64896325936704256093 0.66049822064056939502 0.68509703405346027096 0.68785310734463276836 0.66814764183185235817 0.66436554898093359632 0.65295226130653266332 0.66307692307692307692 0.67156718612172890326 0.67903991068936645269 0.67888487351574599897 0.67699642431466030989 0.69885212694125590817 0.69589436765009284093 0.71630648330058939096 0.72849056603773584906 0.73582587326663156047 0.75036051914757250441 0.76236606487597240569 0.79265091863517060367 0.80684844641724793912 0.80107897153351698806 0.80899795501022494888 0.81387293774496399599 0.80046627542406945896 0.82156509195816804904 0.80259008253371804335 0.77872017216279511361 0.76200227531285551763 0.75659542717049512338 0.70690687693382573111 0.66161224023749714547 0.63831382513433581872 0.63274799395278520758 0.65297725912627169360 0.67565052302423407661 0.66862231534834992142 0.66971826227957570434 0.67437442683086597668 0.67380348702538579522 0.67390654875717017208 0.68103157658163838634 0.68896187119512976610 0.68340966921119592875 0.68580216535433070866 0.67454291728754227867 0.67335379929083077509 0.68231992075097881976 0.67524272923193139150 0.69005997192803368636 0.67449592239845109812 0.66790283701094010755 0.67798195422535211268 0.67646389563106796117 +48 0106000020E610000001000000010300000001000000300000000000002066035AC00000002060D94440FFFFFFFF84035AC0FFFFFF3F39C84440000000A06C035AC000000080ADB14440000000E046035AC0000000C06980444000000020C63B5AC000000060457F444000000000CF515AC0000000E0887F4440FFFFFF3FFC8C5AC0000000A003804440000000E006955AC0000000202B8044400000008059B75AC000000080CE7F4440000000806BD35AC0000000800580444000000020C1FA5AC0000000C06F804440000000800C435BC000000000CE7F4440000000E017805BC0000000E0B27F4440000000A0FF835BC000000080BC7F4440000000E037C35BC0000000C0917F44400000002042C35BC0000000E015A144400000008038C35BC00000002011CA4440000000A011C35BC0000000A085FF4440FFFFFF1FF2C25BC0000000E06C404540000000C019C35BC0FFFFFF1F8E824540000000A0FDC25BC0000000C074A44540000000A0F1C25BC000000060FFC14540000000202DC35BC0000000E0C6FD45400000002040C35BC0FFFFFF3F983C46400000000041C35BC00000006010554640000000A05EC35BC000000040757F464000000000739B5BC000000040057F4640000000A016995BC000000000D57F4640000000A0AA7F5BC0FFFFFF7F5D804640FFFFFF1F1D735BC000000040F27F464000000000F9275BC0000000C0B27F4640000000608C105BC00000004005804640FFFFFF9F32F95AC0000000E0F97F4640000000A08D905AC0000000A0827F46400000002051815AC0FFFFFF1FA57F4640000000E067455AC000000080FA7F464000000000A4425AC0000000C02380464000000060CC035AC000000020A97F4640FFFFFF3FC6035AC0000000E084494640FFFFFFFFDF035AC0FFFFFF9F46174640000000A0CA035AC000000000AB12464000000080C6035AC0000000C02CED4540FFFFFFDFAC035AC0FFFFFF7F7AC0454000000040C1035AC00000002055BD4540000000C090035AC0FFFFFF5F65804540FFFFFFFF90035AC0000000A0AE4E45400000008064035AC000000020FBFF44400000002066035AC00000002060D94440 Wyoming 56 0.58593750000000000000 0.56521739130434782609 0.54029511918274687855 0.55325443786982248521 0.59265175718849840256 0.60441176470588235294 0.68698060941828254848 0.63479262672811059908 0.63962065331928345627 0.70566037735849056604 0.65294771968854282536 0.58617332035053554041 0.66924398625429553265 0.60590500641848523748 0.72190834902699309479 0.76547842401500938086 0.76520681265206812652 0.77518497438816163916 0.85084745762711864407 0.88656387665198237885 0.91246537396121883657 0.82843373493975903614 0.88421528720036182723 0.79766374634960367126 0.78953821005312627707 0.76935817805383022774 0.75702413929560743965 0.73153874136049472535 0.75871886120996441281 0.79531307213474917613 0.80437853107344632768 0.79015721120984278879 0.78468113083497698882 0.78580402010050251256 0.78000000000000000000 0.76095266098206409879 0.76555958693831984371 0.75012906556530717605 0.74398092967818831943 0.74611748818365968940 0.73942644935011347225 0.76994106090373280943 0.80547169811320754717 0.82657539055643321046 0.86684826149655503926 0.90591516218993101424 0.92568034258875535295 0.91464806594800253646 0.93572084481175390266 0.95950920245398773006 0.96363139185124418923 0.94485087225661226787 0.92888568337540569780 0.88914983560356975106 0.80530413317298563200 0.76751990898748577929 0.75904706070457815914 0.69887214292843597165 0.64827586206896551724 0.62348481692839588453 0.63507384579602279335 0.67309994015559545183 0.70234151062800134013 0.68273092369477911647 0.68532531585703668671 0.68639460238438359754 0.67342786490124268319 0.66019359464627151052 0.66917631194516237780 0.66557193207305350849 0.67165394402035623410 0.67002952755905511811 0.68880531706047919771 0.70983555841127055520 0.74399735836596065852 0.74920571441267302095 0.78012419718429671218 0.81254766980228032777 0.80572964954570739848 0.82851012323943661972 0.80597694174757281553 +28 0106000020E6100000010000000103000000010000007E0000000000004056DF52C0000000C07CDB4340000000407CDE52C00000000028DC43400000008068DE52C0000000A0F0DE4340000000007DDD52C000000020B2E143400000008064DB52C0FFFFFFBF99E343400000002060DA52C00000008013E54340000000E0EEDA52C00000006041E64340000000C029D652C0FFFFFF7F98EC43400000006043D052C0000000E036EC4340000000E0D4CF52C000000060D6EC4340FFFFFF1FE7CB52C0000000204BF04340000000802BC952C0000000A0D4F0434000000040B7C852C000000080C9F243400000004071C952C0000000E0A1F74340000000C0F9C852C0000000E057FA43400000004020C752C00000008000FD43400000002070C552C000000020E1FC43400000002061C452C0000000A01DFE4340FFFFFF9FF2C252C000000080F6004440000000C0F7BE52C0000000E058044440000000C038BD52C0FFFFFF5F6907444000000000D4B752C0FFFFFF1FFA0944400000002015B552C0000000C0DA0E4440000000C0C9AF52C000000060E70F4440000000C074AE52C0000000E018134440000000604FAF52C000000020BC16444000000040F6B552C0000000C0C91F44400000002063B852C0FFFFFF7F55264440FFFFFF7FFABA52C0000000C02E28444000000080B4BB52C0000000A0B52A444000000040D6BC52C000000000352C44400000006049BE52C000000080C1334440000000200EC052C0000000404A344440000000C062C152C0000000205B334440000000A0B3C352C0000000C0C43544400000002083C452C000000020633A4440000000A019C452C0000000E0AC4244400000008021C552C0000000E0CA45444000000080FEC752C00000000048484440000000C0B0CB52C0000000E04147444000000040ABCC52C0000000E008494440000000C068CC52C0FFFFFF9FB54A444000000060D9CC52C0000000A0AC4E444000000040B1CC52C0000000602A5144400000002065CD52C00000002048534440000000E0CDCB52C0FFFFFF7FB8554440FFFFFFBF2ACD52C0000000A0CD574440FFFFFF5F0ACC52C000000020A45C4440000000E06ACC52C000000060BB5F4440000000A0ECCA52C0000000C029634440000000605DC852C0000000E0E46244400000008075C652C0000000E051654440000000E0BBC552C000000020206944400000002065C652C0000000806A6B44400000002085C352C000000080836D4440000000C0A3C352C0000000009C6F444000000080BDC452C00000004038714440000000A020C552C0000000C09D734440000000A0B2C852C0000000603E7B4440FFFFFF7FF0C852C0000000801C7D4440000000E07CC752C00000000005804440000000C080C452C0000000005C814440000000E047C252C000000000998344400000002019C052C000000060FC87444000000000DEBD52C0FFFFFFFF918A44400000002050BF52C000000000748A4440000000E009BF52C000000060B58C444000000000D3BC52C0000000204E8E4440000000E091BA52C0FFFFFF9F0C924440000000E03FB752C000000060749A4440000000A079B752C0FFFFFF3F059D444000000060D6B452C0FFFFFF9F2CA44440000000A0D9B252C000000080C6A5444000000000B1B252C000000060EBA7444000000040D4AC52C000000080DCAC444000000020D59752C0000000800E994440FFFFFF7F898F52C0000000A09A91444000000020A98D52C0FFFFFF5FCF8F4440000000606A7952C0000000A0CC7F444000000060617952C0FFFFFF9FFA7A444000000060337A52C000000000AF764440000000600F7B52C00000008066714440000000008F7E52C0000000C010664440000000606D8052C0000000A06A5E4440000000206C8052C000000060195A444000000040498852C0000000E0CF524440000000A06B8752C0000000A04E5A4440000000E06C8952C0000000C072564440FFFFFFBF5A8D52C0000000A0A84B444000000060E09152C000000040D1414440000000A03C9152C0000000A0583B444000000000658E52C0000000E0C3384440000000C0D38752C0000000E0C5394440000000A0A57E52C0000000A068294440000000A0898252C000000060050D4440000000C0E08552C000000020D80E444000000060638552C0000000C0450B4440000000C0388252C000000020AE0B4440000000A0368352C00000004043074440000000A0D88752C0000000C095064440000000E0F98452C00000006065054440000000A0378A52C00000004072F0434000000080FF8A52C000000080ECDB4340000000E03C8F52C0FFFFFFDFDCCF434000000060B89452C00000004036C94340000000A0129552C00000006002C34340000000406B9A52C0000000C070C54340000000A0B29952C00000004052C04340000000207E9D52C0FFFFFF1F9CB6434000000080AA9C52C000000040C3B043400000000027AA52C0000000E0C0A4434000000000DEA752C0000000C008A44340000000C0FFA752C00000006017A04340FFFFFF1F5FB352C0000000805C834340000000004CB852C000000080AF7E4340000000A01BB852C0000000E0707A434000000000FCBD52C0000000405E7C434000000060FFB852C0000000408F8E4340000000C0B0BA52C0FFFFFFBFD395434000000040F2C052C0000000406099434000000080B3C752C000000040A097434000000040A0DA52C0FFFFFF5FFBAF43400000004066E352C000000080C5BE43400000004017E152C0000000A081C843400000008084E452C0000000400ECF43400000004056DF52C0000000C07CDB4340 New Jersey 34 0.79687500000000000000 0.81835748792270531401 0.83541430192962542565 0.86834319526627218935 0.83546325878594249201 0.84264705882352941176 0.86565096952908587258 0.81682027649769585253 0.78714436248682824025 0.87672955974842767296 0.83314794215795328142 0.79844206426484907498 0.82216494845360824742 0.74903722721437740693 0.89704959196484620213 0.97185741088180112570 0.96228710462287104623 0.86852589641434262948 0.88757062146892655367 0.90748898678414096916 0.89972299168975069252 0.86843373493975903614 0.90456806874717322479 0.88193575302461410096 0.91213731099305271761 0.91884057971014492754 0.91017016224772457459 0.89050563841396871590 0.90782918149466192171 0.92456975466861955328 0.93926553672316384181 0.94463431305536568694 0.93030900723208415516 0.93907035175879396985 0.94276923076923076923 0.94795648338723904734 0.95283282165782863522 0.94269488900361383583 0.92777115613825983313 0.95363493135268962413 0.93356715494120074273 0.94990176817288801572 0.96735849056603773585 0.96910654730559943830 0.96827431501361961224 0.96521356230735358873 0.97209559331399364553 0.97691819911223842739 0.97130394857667584940 0.96196319018404907975 0.95424300428402151126 0.94686068011898062545 0.94172376487558600793 0.93934107226732872576 0.95170580416482055826 0.94135381114903299204 0.94078772051377711453 0.93377582593073160994 0.92395524092258506508 0.92231432498854500771 0.91464123735318060240 0.92631657690005984440 0.93634366973160108700 0.92882835690588440719 0.91551246537396121884 0.91330407441372985720 0.91642407737815757348 0.92002270554493307839 0.90942802562085627599 0.91610594894798675638 0.90460559796437659033 0.91001476377952755906 0.89782161356701908749 0.91002117988624735251 0.91438275390348601349 0.90210846719544979893 0.89621879120411722173 0.89312185868225998866 0.89323196736510291118 0.90289392605633802817 0.91252654733009708738 +6 0106000020E6100000010000000103000000010000002800000000000000F96152C000000040E6C2444000000000206152C0FFFFFF1F32D54440000000C0045F52C0000000200F06454000000000F34252C0000000C0A2044540000000406B4052C0000000E09804454000000040483452C0FFFFFFDF46044540000000605C3452C0000000A0A0FF444000000080273152C0000000E04300454000000020683052C00000000052044540000000200A2752C000000080E503454000000040EE2652C000000000E8024540000000A0452552C0000000E0C0024540000000A0962452C000000000D8034540000000C0832052C000000000ED034540000000C0C10852C0000000005E034540000000E01B0652C0000000404A0345400000008061F351C0000000E049024540FFFFFF9F17F351C0000000E088004540000000A07AF251C0FFFFFF3F5ADC444000000000C2F251C0000000E021D24440FFFFFF7F9AF251C000000060F4CC44400000002068F351C0000000A036B54440000000C02CF651C000000040AEB344400000004097F551C000000020C2AB4440FFFFFFDF49F651C0FFFFFFBFA1A94440000000A07FF751C0000000404DA94440000000800A1252C000000060F9A34440000000C0EA1452C0000000C00FA54440000000A03F1852C000000020DBAD4440000000603B1852C0000000A095A3444000000000C62152C0000000C0BDA14440000000600F3A52C0000000408EA24440FFFFFF1FB64652C0000000C09994444000000040D46952C000000020C87F4440FFFFFF9FDD6952C0000000409A81444000000040716E52C000000040D58C4440000000A0A05E52C0000000E0F69A4440000000803E6352C00000004092A54440000000C0DC6252C000000000BFAE444000000000F96152C000000040E6C24440 Connecticut 09 0.88888888888888888889 0.88985507246376811594 0.90919409761634506243 0.91715976331360946746 0.93130990415335463259 0.96029411764705882353 0.97783933518005540166 0.92857142857142857143 0.90621707060063224447 0.96729559748427672956 0.92992213570634037820 0.89386562804284323272 0.98367697594501718213 0.91014120667522464698 1.00000000000000000000 1.00000000000000000000 0.95194647201946472019 0.89926010244735344337 0.96158192090395480226 0.94713656387665198238 0.92132963988919667590 0.91132530120481927711 0.97602894617819990954 0.95869837296620775970 0.97793216183081324070 0.97308488612836438923 0.98021369212504946577 0.97635503819570753001 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 0.99717336683417085427 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 0.99386503067484662577 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 +26 0106000020E61000000100000001030000000100000054000000000000A0B1C95DC0000000A0B83443400000006057D45DC0000000207B434340FFFFFF9FC7E45DC000000000FC59434000000060DAF85DC0000000600F764340000000A0A1FF5DC0000000E0417F4340FFFFFFDF9FFF5DC00000008023884340FFFFFFDF95FF5DC0000000C0988D434000000000A6FF5DC0000000C053944340FFFFFF3FA2FF5DC0000000C0E3A74340000000C0ADFF5DC0000000A0C7B84340FFFFFF3FB0FF5DC00000000040DC4340000000A0B2FF5DC0000000E0BD9644400000008083FF5DC0000000E0A1FE44400000002071D65DC00000004096FE4440FFFFFF7FD5D35DC0000000809FFE4440FFFFFFFFCB8B5DC00000004095FF4440FFFFFF5F25415DC00000000059FF4440FFFFFF5F723F5DC00000000059FF44400100006095FC5CC00000004052FF4440FFFFFFDF88C15CC0FFFFFF7F90FF4440FFFFFFFF30915CC0000000407DFF44400000004072825CC0000000C06BFF4440FFFFFF3F63825CC0FFFFFF9FB67F4440000000A062825CC0000000E0380E4440FFFFFF1F7F825CC0000000E054F443400000008083825CC000000080F5C44340000000E0C7825CC0000000E0EA56434000000060D5825CC0000000201B49434000000040F9825CC0FFFFFFDF9F11434000000000F9825CC0000000E09BCC4240000000C0C2825CC0000000608F7F424000000020BF825CC000000080C26B4240FFFFFF9F57825CC000000020A71B424000000000D6825CC0000000C0D4184240FFFFFFBFD8865CC0FFFFFF5F800F4240000000E034885CC0000000E057054240000000A02E8D5CC0000000E03502424000000020E48E5CC00000002059024240FFFFFF5FA2935CC0000000C0F70742400000004065935CC0000000E0260B4240000000C02D945CC000000020440E4240FFFFFFBFFA955CC00000008099114240000000E051985CC00000004054134240FFFFFF5F5C9C5CC0000000407F0F4240000000C0CF9D5CC000000020F70F4240000000A0E7A15CC0000000A0DA134240FFFFFF9F47A65CC0000000A0B5114240FFFFFF1FBBA75CC0000000A02C1242400000008090AD5CC000000040770D4240000000A08CAE5CC000000080010B4240FFFFFFFF99AE5CC0000000E08507424000000000E1AD5CC000000020B5044240000000C010AF5CC0000000E06BFE4140000000A0B3AC5CC000000040B0F44140000000604AAA5CC0000000E0B3F04140FFFFFF7F58AA5CC0000000207CEF41400000008019AC5CC00000008079EC4140FFFFFFBFA4AB5CC000000040E2E14140FFFFFF5F08AC5CC000000020C6DD4140000000A083AA5CC0000000E0B7D84140FFFFFF3FBBAA5CC00000004005D4414001000000CFA95CC0000000A0C3D2414000000060E6A85CC00000000041CE4140FFFFFFBFBFA95CC000000020DCCA41400000000089A95CC000000080F8C54140FFFFFF5FF8AA5CC00000008004C241400000000041A95CC0000000C0B2B94140000000A0AEA55CC000000080DFAD4140000000E092A55CC0000000C002A7414000000020C3A35CC0000000002F9C414000000000DBA35CC00000000051964140000000C092A45CC0000000C0ED914140FFFFFF7F3CA55CC0000000C0F7904140FFFFFF7F0AA85CC0000000E023914140000000A0A5A85CC000000020308F4140FFFFFFBF11A65CC000000040BC894140000000C082A85CC0000000C05B85414000000080B2A75CC0FFFFFF5FDC7F4140000000E005E85CC000000040DAE5414000000080A2F85CC00000004029004240000000A0354A5DC000000000D67A4240000000E09D755DC0000000008BBA4240FFFFFF5FA79A5DC0000000E080F14240000000A0B1C95DC0000000A0B8344340 Nevada 32 0.75347222222222222222 0.80483091787439613527 0.74006810442678774120 0.81360946745562130178 0.79073482428115015974 0.80294117647058823529 0.91135734072022160665 0.97119815668202764977 0.80295047418335089568 0.98113207547169811321 0.95773081201334816463 0.87147030185004868549 0.84364261168384879725 1.00000000000000000000 0.94852479598242310107 0.92745465916197623515 0.97992700729927007299 1.00000000000000000000 1.00000000000000000000 0.96806167400881057269 0.99445983379501385042 0.95951807228915662651 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 0.90505638413968715897 0.91174377224199288256 0.94983522519223727572 0.97069209039548022599 0.98769651401230348599 0.97205785667324128863 1.00000000000000000000 0.97661538461538461538 0.94354601587768303440 0.92073681272676528049 0.89597315436241610738 0.87246722288438617402 0.92595093405356740941 0.93253558902413864246 0.97170923379174852652 0.98622641509433962264 0.97542566262945409865 0.97965069700368530684 0.95259063554968442683 0.96822765575355712115 0.97894736842105263158 0.98140495867768595041 1.00000000000000000000 0.98122322486555464406 0.94702146474797009406 0.92174540209159754778 0.87136818090317385761 0.85226913095765554782 0.82110352673492605233 0.81714011618611096307 0.79983032238746381874 0.77122630737611326787 0.75727912692131461657 0.75853781447455130442 0.77326451226810293238 0.79227934333469828388 0.79252662825213899075 0.79268292682926829268 0.80685837809511332373 0.80783798165711960434 0.81088671128107074570 0.79225193842004719631 0.79584534871301933141 0.78936386768447837150 0.75120570866141732283 0.72655455427044159039 0.73413293353323338331 0.75904523798292372282 0.77222333311115554667 0.79866870826421674961 0.75591105548276064381 0.74774707954756165400 0.73965669014084507042 0.72074104975728155340 +2 0106000020E610000001000000010300000001000000990000000000002045A15CC0000000A08A83404000000040BBA35CC000000000B4844040FFFFFF1FFCA65CC000000080738340400000000086A85CC0FFFFFF9F4A844040010000603DA95CC000000000AE854040FFFFFF3F71AA5CC000000020FA844040FFFFFFDF79AD5CC000000040348C4040FFFFFFDF5AAD5CC0000000C0A88F40400000006059AB5CC00000004067954040FFFFFFFF78AB5CC000000060BE9C4040FFFFFF7FF6AB5CC0000000E09E9E40400000006052AB5CC0000000404DA240400100004004AF5CC00000006020A74040010000E0FAAC5CC0FFFFFFFF1AAD40400000006058AE5CC0000000A09CB44040000000403CA95CC0000000A0A4B54040010000604EA85CC0000000203EB84040FFFFFFFFB2A75CC000000020FABB4040FFFFFF1F3AA65CC00000008038BE40400000008085A55CC0000000A034C1404000000020D5A15CC0000000A0AFC740400000008086A25CC0000000404DCA404000000040B0A15CC000000060A1CF4040FFFFFFFF90A15CC0000000802ED540400000000048A25CC00000002063D74040FFFFFF3FAC9F5CC000000020ABDA4040FFFFFF9F9BA05CC00000008020DF4040000000C03DA05CC0000000E0C6E24040000000204DA15CC0000000C0BAE9404000000020B3A05CC0000000E0C4EB4040000000804AA15CC0FFFFFFBF73EE404000000060D59F5CC00000000067F640400000000097A15CC000000020E8F94040000000601DA15CC0000000A086FB404000000080679B5CC000000060D103414000000060169B5CC000000040060A4140FFFFFF3F339A5CC000000040230D4140000000E09B945CC00000008015124140000000A036925CC000000060EA154140000000200A8F5CC0000000A0D5174140FFFFFF5F8B895CC0000000E02B224140FFFFFFFFF6875CC0000000C0E4224140000000C088885CC0000000A042284140000000C0C4895CC000000040102B4140000000609A8B5CC0000000A0BE2E4140000000A073905CC0000000A0E63341400000004016925CC000000040BE3441400000004055935CC000000060C6374140000000003D955CC0000000E0383A4140000000C00B985CC0FFFFFF5FD63A41400000004084985CC0000000C0103D41400000000011985CC0000000C0AD444140FFFFFF3F2C9A5CC000000020B74A4140000000A0BE9B5CC000000080AA4C414000000080F99A5CC000000080314E414000000000C09D5CC0FFFFFFDFDC5A414000000000CF9F5CC000000000545F4140000000A095A15CC000000020DC5F4140000000C0A3A25CC00000002046614140FFFFFF5F71A45CC0FFFFFF3F7A6A4140FFFFFFFF17A85CC00000006011704140000000A04CA85CC000000020B275414001000080B1A75CC000000020C8784140FFFFFF1F6AA85CC000000000B37F414000000080B2A75CC0FFFFFF5FDC7F4140000000C082A85CC0000000C05B854140FFFFFFBF11A65CC000000040BC894140000000A0A5A85CC000000020308F4140FFFFFF7F0AA85CC0000000E023914140FFFFFF7F3CA55CC0000000C0F7904140000000C092A45CC0000000C0ED91414000000000DBA35CC0000000005196414000000020C3A35CC0000000002F9C4140000000E092A55CC0000000C002A74140000000A0AEA55CC000000080DFAD41400000000041A95CC0000000C0B2B94140FFFFFF5FF8AA5CC00000008004C241400000000089A95CC000000080F8C54140FFFFFFBFBFA95CC000000020DCCA414000000060E6A85CC00000000041CE414001000000CFA95CC0000000A0C3D24140FFFFFF3FBBAA5CC00000004005D44140000000A083AA5CC0000000E0B7D84140FFFFFF5F08AC5CC000000020C6DD4140FFFFFFBFA4AB5CC000000040E2E141400000008019AC5CC00000008079EC4140FFFFFF7F58AA5CC0000000207CEF4140000000604AAA5CC0000000E0B3F04140000000A0B3AC5CC000000040B0F44140000000C010AF5CC0000000E06BFE414000000000E1AD5CC000000020B5044240FFFFFFFF99AE5CC0000000E085074240000000A08CAE5CC000000080010B42400000008090AD5CC000000040770D4240FFFFFF1FBBA75CC0000000A02C124240FFFFFF9F47A65CC0000000A0B5114240000000A0E7A15CC0000000A0DA134240000000C0CF9D5CC000000020F70F4240FFFFFF5F5C9C5CC0000000407F0F4240000000E051985CC00000004054134240FFFFFFBFFA955CC00000008099114240000000C02D945CC000000020440E42400000004065935CC0000000E0260B4240FFFFFF5FA2935CC0000000C0F707424000000020E48E5CC00000002059024240000000A02E8D5CC0000000E035024240000000E034885CC0000000E057054240FFFFFFBFD8865CC0FFFFFF5F800F424000000000D6825CC0000000C0D4184240FFFFFF9F57825CC000000020A71B424000000020BF825CC000000080C26B4240000000C0C2825CC0000000608F7F4240FFFFFFBF8C395CC0000000E0847F424000000040AC225CC0000000C0BE7F424000000000230F5CC0000000C06C7F4240000000C0BFD65BC00000006038804240FFFFFFDF51AF5BC0000000A05180424000000000F09E5BC0000000C08080424000000020E69C5BC0000000A0F17E424000000000C57F5BC000000000FC7E4240FFFFFF7F0F435BC000000000927F4240FFFFFF3F05435BC00000006092FF4140000000A0F1425BC0000000C0307A4140FFFFFF9F12435BC000000020BE4B4140000000A02E435BC00000008041E44040FFFFFF9F31435BC0000000C0409A4040000000203F435BC000000000C6634040000000E020435BC0000000609238404000000060E1425BC0FFFFFF9FE5573F40000000A0EC9C5BC0000000406A563F400000008090C45BC0000000A0E5553F40000000809BD75BC0000000C0726E3F400000002004555CC00000006093054040FFFFFFBF8AB45CC0000000C0593E4040FFFFFF1FC0B35CC0000000E0D84E40400000002029AE5CC000000080435C4040FFFFFFDF8FAD5CC000000060135E4040FFFFFF3F5EAC5CC000000080E55E4040FFFFFF1F9AA65CC000000060F55C40400100004093A65CC000000000305E4040000000208EA45CC0000000E0625E40400000004092A45CC0FFFFFF1FD85F404000000080D6A35CC0000000A0DB5F404000000020E4A35CC0000000E05E614040000000C0B3A25CC0FFFFFFBF5E614040000000C0B6A25CC000000040B6624040FFFFFF3FE0A15CC000000020BC624040000000E031A25CC000000040DD644040000000C0A0A15CC0000000C0A9674040FFFFFF5F7B9D5CC000000060356C404000000040719E5CC000000080CA774040FFFFFF3FED9D5CC0000000E0267D40400000002045A15CC0000000A08A834040 Arizona 04 0.52083333333333333333 0.50241545893719806763 0.48694665153234960272 0.47485207100591715976 0.49201277955271565495 0.53235294117647058824 0.57617728531855955679 0.53225806451612903226 0.53108535300316122234 0.60125786163522012579 0.54505005561735261402 0.49172346640701071081 0.54810996563573883162 0.58857509627727856226 0.63025737602008788449 0.65666041275797373358 0.68369829683698296837 0.63574274331246442800 0.67005649717514124294 0.72907488986784140969 0.72299168975069252078 0.65879518072289156627 0.73405698778833107191 0.71589486858573216521 0.70126685737637923989 0.70227743271221532091 0.69331222793826671943 0.67297198981447799200 0.67366548042704626335 0.69022336140607835958 0.69703389830508474576 0.70369104579630895420 0.69132149901380670611 0.68059045226130653266 0.67815384615384615385 0.67921199647162599236 0.67317890036282444879 0.66778523489932885906 0.65649582836710369487 0.69592617600720234076 0.71982669692593356715 0.75500982318271119843 0.78207547169811320755 0.78760751272599613832 0.78977727928216631950 0.78218112432115074123 0.76364138693189667081 0.77032339885859226379 0.76239669421487603306 0.77566462167689161554 0.78424938474159146842 0.77096229600450196961 0.76869816083663901911 0.73441588941823793867 0.73764162288752452687 0.73293515358361774744 0.73591643127431647391 0.72177862062082044116 0.69102534825302580498 0.65793310284500354063 0.64224522231267201613 0.64373877917414721724 0.65379890555783047314 0.63317618299284092893 0.63360583744341598541 0.64764836892440717935 0.64588224246408113438 0.64564412045889101338 0.63998763906056860321 0.64437146213820356723 0.64094147582697201018 0.62937992125984251969 0.62044986872915631874 0.62989933604626258299 0.63932260955705457805 0.63882778999755604435 0.65226915061035260091 0.63661431951968396143 0.62061932134248099388 0.61335460680751173709 0.60825621966019417476 +27 0106000020E6100000010000000103000000010000007B000000000000E0F11152C0FFFFFF7F355C454000000000331D52C000000000E65C4540000000E09B1D52C0000000C0955F4540000000E0B41E52C00000000079614540000000C07E2052C00000002063624540000000C0DD2052C0000000C00365454000000040852252C0000000206167454000000000732352C0000000E0266E4540000000209E2152C0000000E0CD744540FFFFFFDF522152C0000000C0CD794540000000804D2052C0FFFFFF9F957B4540000000E0521E52C0000000A0EF7C4540000000C0491D52C000000060F07F454000000000951D52C000000040F1854540FFFFFF5F691C52C0000000601B8A454000000060091C52C000000060DF8E4540000000C0FB1C52C000000000F693454000000080E81B52C000000000B89D4540000000E0C81952C0000000E055A74540000000E0481A52C0FFFFFFDF62A94540000000607A1952C000000060EBAC4540FFFFFF1F681A52C00000004043B04540000000C0631952C0000000A07DB44540000000C0821852C00000000006BE4540000000404F1952C0000000E03CC2454000000000EF1752C00000004041C94540000000E0581552C0FFFFFF1F74CC4540000000007D1352C00000002067D9454000000060AC1052C0000000001CDE4540000000C00D0E52C00000006014E04540FFFFFF3F380D52C000000040DDE1454000000000DC0B52C0000000809BE6454000000060EA0A52C0000000207EF04540000000C0D00752C0FFFFFFFF5EF44540FFFFFF5F460752C00000006034F8454000000040E60552C0000000409DFA4540000000E03F0752C000000040FCFC454000000060100752C0000000E09CFE4540000000A07B0552C0FFFFFF3F22014640000000E0F30452C0000000C017044640000000A0400252C0000000E0A90A4640000000401B0252C0000000E04A0C4640FFFFFFDF320352C000000080D90C4640000000C0430252C0FFFFFF7F720F464000000060E40252C00000000004144640000000E0D20352C0000000804F174640000000E0DE0252C000000020FE1D4640FFFFFF7FD70352C0000000A076214640000000204D0252C0000000E051264640FFFFFF5FACFF51C000000020EB294640000000A01EFC51C0FFFFFF5FB1294640000000E071FB51C0000000C0032B46400000004075F551C0000000C00C2C4640FFFFFF1F96F451C0000000A0D62C4640000000A015F351C0FFFFFF9F2A3146400000002017F151C0000000E0F7324640000000C059EB51C0FFFFFF9FEC354640000000200AEA51C0FFFFFF7F54384640000000C07BE951C0000000000C3C464000000000C5E851C0000000A0033D4640FFFFFF1F57E751C0000000C0BA3C4640000000E092E551C0000000204B3F464000000080D8E451C0000000C04B434640000000E0E1E551C0FFFFFFFFF7444640000000C0EFE551C0FFFFFFFF8B4646400000008062E251C0000000A0184A4640000000207EE351C0000000405B4C4640FFFFFF3F62E451C00000002096514640000000C0B5E551C00000002045534640000000E0EBE651C000000080C2564640000000206CE851C0000000C0EE5E4640000000C05FE551C000000020BB63464000000020D6E451C0000000C0716846400000000070E051C0000000E026734640000000E01DE151C0FFFFFF7FC978464000000040A6E251C0FFFFFF7FFE7C4640000000C05FE051C000000000B4814640FFFFFFDF38E051C000000000A9874640000000A093DB51C000000020F88E4640000000C0CAD951C000000000F4994640000000409CDC51C000000080369E464000000040C2D851C0000000C0109E4640000000C00DD351C0FFFFFFBF8FA5464000000020D4C951C000000040749E4640000000A0A1C551C00000000095A6464000000080DEC151C0000000E090554640FFFFFFBF94C051C0000000401B2446400000002009BF51C00000006042E54540000000005CBE51C0000000202FC94540000000C03FBD51C00000004033C84540000000A0CEBC51C00000008041C64540000000A0BEBD51C0000000A015C44540000000E06CBD51C00000006016C245400000008029BE51C0000000402DBC45400000000091BD51C0000000E011B84540000000A0B1BE51C000000080B3B24540000000A017BE51C0FFFFFFDFE2AE4540000000C000BA51C000000080A7A64540FFFFFF7FB3B951C0000000C0F5A34540FFFFFFBF13B451C000000000199E4540FFFFFFDF2FB551C0000000005D944540000000406DB451C0000000A0968F4540000000E0A7B851C0000000E0559045400000002002B851C000000000FC8C454000000080FAB951C000000060BE8A454000000080C4B851C0000000A086874540FFFFFF3FE6B351C0000000C0798B45400000006004AF51C0000000A082874540000000E01EB451C000000000F96E4540000000606AB651C0000000E0816E4540000000E082B951C0000000408271454000000060FFBA51C0000000A049714540FFFFFFBFA8C151C000000040F06C4540000000603AC451C000000000F166454000000020C0C751C000000080BA684540FFFFFF9F9ECB51C0000000405367454000000040F5CB51C0000000C08C5E4540000000206CCF51C0000000E0295F4540FFFFFF9F2FD051C0000000E0EC5C45400000008069D251C0000000E068594540000000E0B0F951C000000000475A45400000008090FB51C000000000835A4540000000E0F11152C0FFFFFF7F355C4540 New Hampshire 33 0.59548611111111111111 0.62512077294685990338 0.63337116912599318956 0.63165680473372781065 0.66453674121405750799 0.70000000000000000000 0.68975069252077562327 0.61866359447004608295 0.59536354056902002107 0.67044025157232704403 0.62402669632925472747 0.56377799415774099318 0.60824742268041237113 0.54621309370988446727 0.61268047708725674827 0.65916197623514696685 0.67579075425790754258 0.65452475811041548093 0.68757062146892655367 0.71090308370044052863 0.70581717451523545706 0.64963855421686746988 0.68204432383536861149 0.65790571547768043388 0.67552104617899468737 0.70517598343685300207 0.72378314206569054214 0.69116042197162604583 0.71423487544483985765 0.73379714390333211278 0.75000000000000000000 0.75085440874914559125 0.74983563445101906640 0.75125628140703517588 0.74676923076923076923 0.75036753895912966774 0.75579123639408317053 0.76484254001032524522 0.75375446960667461263 0.77447670492910195814 0.77243655869610068083 0.76542239685658153242 0.77396226415094339623 0.77637352992803229770 0.78192597340169844576 0.77484221341552913548 0.77248238707003729797 0.79365884590995561192 0.79109274563820018365 0.79611451942740286299 0.80038282745419742959 0.79708979821529061822 0.79906238730616660656 0.79889955042608870697 0.82543198936641559592 0.82673492605233219568 0.84309545381868571124 0.84709052799680606847 0.83905001141813199361 0.82305160994709876286 0.79989921308679303795 0.77472321962896469180 0.79388005807244164836 0.77366858739305046272 0.76079318964934801703 0.78016507271059871610 0.78279650671424546906 0.77802342256214149140 0.77556467018766153500 0.79245434155719320731 0.79170483460559796438 0.82017716535433070866 0.80276260081837318763 0.81709145427286356822 0.82296334732770413699 0.81668110822279988447 0.79826464208242950108 0.78221500792052099427 0.77359540144631930280 0.77870085093896713615 0.79418234223300970874 +32 0106000020E610000001000000010300000001000000A0000000FFFFFF3FB9AE58C0000000201AF84640000000A069C058C00000004044F84640FFFFFF7FE2ED58C000000000ADF846400000006006F858C0FFFFFF9FC6F84640FFFFFF5FE52059C0000000C05EF8464000000080268059C000000000A4F84640000000808ABC59C00000006088F84640000000E0A3BF59C00000006076F846400000004019035AC000000000B4F84640000000C007035AC0000000C0F323474000000040F5025AC0000000A0714547400000004001035AC0000000C04B524740FFFFFF7FE8025AC000000080BAAA474000000020FF025AC00000004033B34740FFFFFF5F11035AC0FFFFFFFF01004840000000C04D035AC0000000200C3248400000002051035AC000000080A95248400000000000045AC000000000008048400000000000BC59C0000000E0FF7F4840FFFFFF7F658159C0000000000080484000000000006059C0000000000080484000000000000C59C0000000E0FF7F4840000000201FE258C000000000008048400000000000C058C0000000000080484000000000007C58C0000000E0FF7F484000000080A94E58C0000000000080484000000060D34D58C0000000A04677484000000080394B58C0000000C0D76F484000000060EF4A58C0000000E0016B484000000060864B58C0000000006468484000000000854A58C0000000A0BA674840000000401C4B58C00000008098664840000000406B4958C0000000E0FD634840FFFFFFBFE34858C000000040BC614840FFFFFF7F714958C0FFFFFFBFB960484000000040754858C0FFFFFF5FA55F4840000000009B4858C0000000E0F55C484000000040064758C000000060B35A4840FFFFFFFF724758C000000040FE58484000000060324658C00000008057564840000000C0DD4658C0000000A0A250484000000060224858C0000000A09D504840000000E0D84758C0000000E0754F4840000000603D4958C0000000E0984E484000000060FD4858C000000040204B4840000000201A4A58C000000040B54A4840000000C0B64958C0FFFFFFDF53494840000000E0B94A58C0000000C0F8474840000000805C4958C0000000C057464840000000E03E4A58C0FFFFFF9FC5454840FFFFFF9FEE4958C0000000C0EA44484000000000E64858C0000000E06F44484000000080784958C0000000C04C42484000000080974858C0000000E037424840000000402B4958C0000000801438484000000060A24758C000000080F337484000000000D34758C0000000A043354840000000E0AE4958C0FFFFFF5FB635484000000000914958C0000000207B344840000000E03D4858C0FFFFFF1F3634484000000060244A58C000000040B1314840FFFFFF7FA14858C0000000E034314840FFFFFF3F8A4858C000000020AD2F4840FFFFFF5F9A4958C0000000607E2E4840000000A05E4858C0000000E0452E484000000020C14858C0000000A0BA294840FFFFFFFF2E4758C0000000E0F328484000000060774858C000000060CE27484000000060524758C000000080DD26484000000080414758C000000040BF254840FFFFFF9F544858C0FFFFFFDF8225484000000080304758C000000000A1244840000000A0204758C0000000009223484000000040B94858C000000020DB21484000000060E64758C0000000202D21484000000020244858C0FFFFFF9FE41D4840FFFFFFFFF74658C000000040311D484000000000EC4858C0FFFFFFFF621C484000000040134758C0FFFFFF5F931A4840000000C0594858C0000000C0141A484000000060C34858C000000060F818484000000040B34858C0000000606E16484000000020C64858C0000000007A154840000000E0674758C00000000062144840000000E0B64858C000000040FF12484000000060B74758C0FFFFFFFF4612484000000000C74758C0000000C0E50E4840000000C0504658C000000040ED0C484000000060E94558C0000000A00109484000000020454458C0FFFFFFDF2A064840000000800D4358C0000000A03BFA474000000060F54058C0000000207EF54740000000204B4158C00000006013F04740000000C0FF3F58C00000006063EF474000000020853E58C0000000A0FDE9474000000040F23E58C0000000C0A3E7474000000040473D58C000000000B1E5474000000040A03B58C000000060BBE14740FFFFFF5F173B58C00000004068DB4740FFFFFF7FE63858C0FFFFFFFF43D64740000000E0DE3758C000000080C1CE4740000000E0843658C000000060F3CC474000000080EE3658C00000008010C84740FFFFFF3F533658C000000040B5C54740000000A00F3758C0000000A0BCC24740000000007B3658C00000002015C04740FFFFFFDF713758C00000008014BB474000000000C03658C0FFFFFF5FE8B74740000000207B3758C0000000E0E0B44740000000A0603658C0000000E058B44740000000E0B93558C0000000A02BB14740FFFFFFDF6A3658C0000000A034AE474000000080A33558C000000080CFAB4740000000402B3658C0FFFFFFBF45A8474000000040973558C0000000E09EA54740000000605A3658C000000020E1A04740000000A08C3558C000000040C89E4740FFFFFF5FDF3458C0000000A0C5954740FFFFFFFFAE3558C00000000072934740FFFFFF1F673458C000000080DB8B4740FFFFFF1FE73458C000000040188A4740000000C09F3458C00000000059844740000000A06F3558C00000006050814740FFFFFF3FBF3458C0000000C07180474000000020403458C0000000E0227C4740000000A0C13258C0000000601E7C4740000000404C3358C0000000405A7A4740000000C0843258C0FFFFFFDF5F79474000000080673258C0000000005377474000000040D03058C0000000A0D8774740000000606B3058C0FFFFFFDF1E764740000000E0C53158C000000080066F474000000020253158C0000000A0256C474000000060FF3258C0FFFFFFDFF1674740FFFFFFDFEB3158C0000000A09461474000000020FF3158C0000000A0815A474000000000C63258C000000040E4564740000000808D3258C0FFFFFF9F9D504740FFFFFF5F2C3258C000000020E44F4740FFFFFFDF523158C000000080CD4C4740000000400E3058C000000020594B4740000000805B2F58C000000000A73E474000000000BB2D58C000000040003C4740000000A0652D58C0000000C0AE36474000000020062C58C0000000E0C434474000000040B62958C000000040032E474000000020542758C0000000C0E82C4740000000A0822658C0000000000E2B4740000000E0422658C0000000808E1E4740000000C0822558C000000000941B4740000000409A2558C0000000209218474000000040882458C000000000AF164740000000204D2358C0000000A03B0C474000000000DB2458C0000000A0BA024740FFFFFFDFEE2358C0FFFFFFFF4EF94640FFFFFFBF422458C00000004092F7464000000080E84E58C0000000A0E0F74640FFFFFF1F9D7E58C00000006026F74640000000C0EA8058C0000000803CF74640FFFFFF3FB9AE58C0000000201AF84640 North Dakota 38 0.33159722222222222222 0.30048309178743961353 0.21225879682179341657 0.26035502958579881657 0.23322683706070287540 0.26470588235294117647 0.37673130193905817175 0.26958525345622119816 0.34351949420442571128 0.35471698113207547170 0.35483870967741935484 0.34566699123661148978 0.45446735395189003436 0.42682926829268292683 0.60640301318267419962 0.64477798624140087555 0.63625304136253041363 0.61923733636881047240 0.84406779661016949153 0.81662995594713656388 0.67091412742382271468 0.65542168674698795181 0.65309814563545906829 0.54985398414685022945 0.54597466285247241520 0.56480331262939958592 0.58963197467352592006 0.56602400873044743543 0.56868327402135231317 0.67630904430611497620 0.59286723163841807910 0.62235133287764866712 0.54339250493096646943 0.72864321608040201005 0.65907692307692307692 0.62099382534548662158 0.68741278258442645828 0.64713474445018069179 0.61787842669845053635 0.61197389151474229124 0.62966783577470600371 0.63143418467583497053 0.69226415094339622642 0.76829910479199578726 0.98894407947444319821 0.89828269484808454425 0.87498273242160519409 0.78427393785668991756 0.73771808999081726354 0.83190184049079754601 0.76547260960714611248 0.65077578583487418603 0.74590695997115037865 0.73743541568811648661 0.72067852395721248180 0.70005688282138794084 0.68277993924212545968 0.65505539474997504741 0.61954784197305320850 0.53088682467613612696 0.55653758188936698066 0.59395571514063435069 0.60566578565312883892 0.61784529422035969967 0.60232416728599418958 0.62337875016376260972 0.59736438476226249726 0.63234942638623326960 0.58427913248679626924 0.60789810958026273630 0.59320610687022900763 0.61683070866141732283 0.61775349464273043355 0.63706242117036719735 0.68656540402849191000 0.65933479970672532160 0.67296159244608906469 0.64255959947587663544 0.66534396439829408492 0.71539392605633802817 0.73331310679611650485 +30 0106000020E6100000040000000103000000010000009C000000FFFFFF9FDCF053C0FFFFFFDF35224540000000A06EDC53C0000000C0AB354540FFFFFF3FBAD653C0000000E0273F4540000000401EC953C0000000208B494540000000C0D0C253C0FFFFFFFF7E5945400000002001B753C0000000C07665454000000060F4BB53C0FFFFFFBFB17C4540000000A083B853C0000000C0DA824540FFFFFFDF40BB53C00000006085884540FFFFFF1FEDC353C0FFFFFF1F978B45400000002088C253C0000000008592454000000080FFC353C00000002053A2454000000000C19D53C0000000A09BAF454000000060817F53C0FFFFFF1FC9AE4540000000A0B26F53C0FFFFFFDFE4AA454000000000DD6453C000000000E99E4540000000A02A5853C00000008048A34540000000C08C3A53C0000000E0A6A34540000000802D2F53C000000080DCAB4540000000C0002E53C00000006064A9454000000060AD2753C0000000C000B54540000000A01E1D53C00000006015C0454000000080470E53C000000040ECC64540000000C0D50B53C0FFFFFF5F0AD14540000000602F0D53C00000008060D74540000000C0610F53C000000060E3EA4540000000A06B0C53C0FFFFFF7FCAF4454000000060480853C0000000A050F74540000000C0A10853C0FFFFFF9FAF014640000000C0EB0C53C0FFFFFFBF63084640000000C0051353C0FFFFFFFF5E054640000000E03E1753C000000020950C4640000000604BF652C000000060F23146400000000093F052C0FFFFFF9F3C424640000000A011D552C0000000C0C06746400000002001BE52C0000000006B7946400000008022AF52C000000040167F464000000060678152C0FFFFFF9FD27E4640FFFFFFDF165652C0000000E0C7804640000000C0795652C000000000B07D4640000000C08E5552C0000000405E774640000000A07E5852C0000000C0876C464000000080A55752C0FFFFFF7FD768464000000000F15452C0000000E04D664640000000C0E85752C000000060B25C4640FFFFFFDFF25652C0FFFFFFDF14574640000000C0E75752C000000040A354464000000040B75752C0000000E030514640000000C0765852C000000060544F4640FFFFFF3FCA5752C0FFFFFF9F204A464000000080495652C0FFFFFFFFE6464640000000A06E5552C000000000AB454640000000C0CC5252C00000002066374640000000203A5352C0000000E0E633464000000040225552C0FFFFFFFF042F464000000080915352C0FFFFFF9F4A214640000000402D5852C0000000C0C0194640000000C07A5852C0000000E00516464000000080215A52C0FFFFFF1F6E11464000000020305A52C0FFFFFF9FA30D4640FFFFFF9FE15B52C0000000002C08464000000080EE5B52C000000000D7054640000000E0275A52C00000004053024640000000E0BD5A52C0000000607BFE454000000020F85952C00000008016F5454000000020095852C0000000C065F14540000000C0A65852C0FFFFFF1FF8E6454000000000015752C000000080A1E34540FFFFFFDFDA5652C0FFFFFFDFD4E0454000000080C55752C0000000806BDB454000000000195B52C000000000E7D0454000000000CD5A52C0000000808CCA454000000020DE5852C000000080D7C84540000000E04D5752C000000020B6CE454000000060745352C000000080F4CF4540000000E0D95252C0000000A04ECF4540000000400F5252C000000060EBCB4540000000A0AD5252C00000008098C94540000000E0AA5052C0000000A097C7454000000020494F52C0FFFFFF3FA2C1454000000080085052C0000000A0C7A7454000000060B15152C0000000005978454000000000EC5152C0000000A0236B4540000000C0FB5252C0FFFFFF1FD866454000000020435152C0000000E0AA5F4540000000608B5052C0000000407C5F4540000000407B5652C0000000209940454000000020F45F52C000000040E7094540000000C0045F52C0000000200F06454000000000206152C0FFFFFF1F32D5444000000000F96152C000000040E6C24440000000C0DC6252C000000000BFAE4440000000803E6352C00000004092A54440000000A0A05E52C0000000E0F69A444000000040716E52C000000040D58C4440FFFFFF9FDD6952C0000000409A81444000000040D46952C000000020C87F444000000020F37152C0000000C07B71444000000040FE7252C0000000A0866A4440000000E0E57A52C000000000BF664440000000206C8052C000000060195A4440000000606D8052C0000000A06A5E4440000000008F7E52C0000000C010664440000000600F7B52C0000000806671444000000060337A52C000000000AF76444000000060617952C0FFFFFF9FFA7A4440000000606A7952C0000000A0CC7F444000000020A98D52C0FFFFFF5FCF8F4440FFFFFF7F898F52C0000000A09A91444000000020D59752C0000000800E99444000000040D4AC52C000000080DCAC4440000000A029AD52C0000000E0FEAF4440000000C069AF52C0000000C065B34440FFFFFF3F63AF52C0FFFFFFFF02B644400000008055B052C0000000000CB74440FFFFFF9FC3B252C00000008004B74440FFFFFF3F53B752C00000006039B94440000000A052B952C000000000E8B844400000006088B952C0000000601CBB444000000080B5BB52C000000040FCBD44400000002038BE52C0000000C0E3BD444000000040FAC052C0000000400DC5444000000060A1C152C00000004069C84440000000007FC452C0000000A05CCD444000000060A9C452C00000008076CE4440000000404FC352C0000000A090D14440FFFFFFDF35C452C0000000E07CDB4440000000C0ABC352C0000000E001DD444000000000F3C352C00000000095E24440000000203EC652C0FFFFFFBFB4E34440000000C038C652C00000000008E644400000008022C552C00000002033E844400000000090C752C0FFFFFF7F1FEB44400000004002C852C000000020AFEC4440000000C083C952C0000000A087ED4440000000A0FCCA52C00000008012EF44400000004050D052C00000006034EF4440000000402ED252C0FFFFFF3F48F94440000000C0C3D452C00000004008FB44400000008025D652C0000000C012FF44400000004086D852C000000060C7FF444000000020BADE52C0FFFFFF3F86FF444000000060BB0653C000000020EDFF4440000000C04D0953C00000000013004540FFFFFF9F1C2453C0000000406000454000000000703B53C0FFFFFF9F5000454000000040023E53C0000000405F004540000000A03D6753C0FFFFFF5FD7FF4440000000E0B26F53C0FFFFFF5FA6FF444000000000178D53C0000000E0C2FF4440FFFFFFDF8A9353C0000000E0EAFF4440FFFFFF3FCDBA53C000000000F9FF444000000080D2C353C00000000024004540000000C034E753C00000006011004540000000C0C2F053C0FFFFFFFF63004540FFFFFF9FDCF053C0FFFFFFDF3522454001030000000100000024000000000000202B7052C000000000184C444000000080647B52C0000000A05D474440000000A0C17052C0000000001F4F4440000000A0FB7052C0000000A083514440000000C02C7652C0000000008553444000000040A27A52C0FFFFFF7FC5504440000000C0557852C0000000808A4B444000000000498052C000000040634A4440FFFFFFFFCC8152C000000060D2514440000000802F7D52C0000000C0A05E444000000060907952C0000000A00466444000000020477052C0000000C0F5644440FFFFFF5FFE6F52C0000000A0256C4440FFFFFF1F516652C0000000009773444000000020A05E52C00000004098704440000000E0965B52C0000000201376444000000000C24D52C00000002052734440000000200D4952C000000020C4794440000000E0634152C000000060F27B4440FFFFFF3F732852C000000080977D444000000000521452C0000000201A93444000000020061252C0000000603B92444000000040B41652C0000000E0178E444000000020AD1A52C00000000052834440000000C04A2352C000000080A87B4440000000E0BC2652C0000000A0DD73444000000040801E52C000000020C4754440000000A0C91252C0000000C00F834440000000800B0D52C0000000E083844440000000A0F50452C0000000800F804440000000E025FB51C0000000C0D98A4440000000E0B5F751C0000000208689444000000000D4FA51C000000080E6834440FFFFFF5F622152C00000000050684440000000C0115B52C000000000A3544440000000202B7052C000000000184C44400103000000010000000C000000000000A0C85252C0FFFFFFFF2950444000000040A95252C00000006005514440FFFFFF7F6A4F52C00000006005514440000000404B4352C0000000A068564440000000A0263852C0000000205E5E444000000000173252C0FFFFFFDFCB614440000000807F3052C0000000404962444000000040ED3052C0000000C00F614440000000A03B3D52C0000000A09759444000000020024252C0FFFFFF3FEB554440000000A0F74F52C0FFFFFF9F0A504440000000A0C85252C0FFFFFFFF2950444001030000000100000008000000000000C0308F52C000000040C1404440000000C0308F52C0000000A0D644444000000060AB8A52C000000020EC4F4440000000A0B08452C00000008021534440000000A0D08352C000000060014D4440FFFFFFFFE58752C0000000A0B6454440FFFFFF5F6B8C52C0000000A056414440000000C0308F52C000000040C1404440 New York 36 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 1.00000000000000000000 0.93087557603686635945 0.88303477344573234984 0.99245283018867924528 0.91768631813125695217 0.84615384615384615385 0.85567010309278350515 0.75032092426187419769 0.86880100439422473321 0.96185115697310819262 1.00000000000000000000 0.96585088218554354013 0.97457627118644067797 0.97522026431718061674 0.95789473684210526316 0.89542168674698795181 0.90547263681592039801 0.85815602836879432624 0.87617490805067429506 0.90062111801242236025 0.90819153146022952117 0.87886504183339396144 0.89750889679715302491 0.93482240937385573050 0.95162429378531073446 0.95283663704716336295 0.94214332675871137410 0.93592964824120603015 0.94461538461538461538 0.95677741840635107321 0.95506558749651130338 0.94398554465668559628 0.93516090584028605483 0.96668917398154400180 0.94965958324736950691 0.96011787819253438114 0.97716981132075471698 0.97209057398630858346 0.95817977888158948886 0.95288419198590929106 0.96076806188700096698 0.94825618262523779328 0.93583562901744719927 0.91809815950920245399 0.90484003281378178835 0.89195272931907709623 0.89174179588892895781 0.89539018989465208347 0.89803152098234065447 0.89527872582480091013 0.89186164259446783563 0.88965964667132448348 0.86919387988125142727 0.85824967717748989878 0.85149435980928014885 0.87204518252543387193 0.89126307560585191527 0.88000698445957744020 0.86443483548408891291 0.86332372592689637102 0.86771840861426738035 0.87434273422562141491 0.85627598606585009552 0.86072839901740895012 0.86234096692111959288 0.85007381889763779528 0.83660919132429811490 0.84081768639489778920 0.85091277890466531440 0.85120753627052367304 0.86325549742673642125 0.85838890735923962998 0.88103096606712404969 0.89297241784037558685 0.88827366504854368932 +36 0106000020E6100000010000000103000000010000009A00000000000020785E53C0FFFFFF7F19DC434000000080288653C0000000E0D8DC4340000000A06D9553C000000040ACDC4340000000A0A49853C0FFFFFFDFA0DC4340000000605AB453C0000000408CDC4340000000208CBB53C0000000C072DC43400000000066D953C0000000E00FDC4340FFFFFF1FCCDE53C0000000602FDC4340000000A0FBF053C0000000A061DC434000000080C8FA53C0000000005DDC434000000060791B54C00000002021DC4340000000E0902154C0000000E04DDC434000000040972154C0FFFFFFFFAAFA4340000000609C2154C000000080E902444000000020AE2154C000000020CB14444000000080852154C0000000609433444000000080922154C0000000A0463D4440000000E06B2154C0FFFFFFBF8D514440FFFFFF1F502154C000000060536D4440000000405A2154C0000000C0D7724440000000207B2154C0000000C095904440000000004B2154C0000000409FBE4440FFFFFF5F8B2154C0FFFFFFBF5DBF444000000060762154C000000060E4EC4440FFFFFFBF542154C00000000050FE4440FFFFFF9FDCF053C0FFFFFFDF35224540000000C0C2F053C0FFFFFFFF63004540000000C034E753C0000000601100454000000080D2C353C00000000024004540FFFFFF3FCDBA53C000000000F9FF4440FFFFFFDF8A9353C0000000E0EAFF444000000000178D53C0000000E0C2FF4440000000E0B26F53C0FFFFFF5FA6FF4440000000A03D6753C0FFFFFF5FD7FF444000000040023E53C0000000405F00454000000000703B53C0FFFFFF9F50004540FFFFFF9F1C2453C00000004060004540000000C04D0953C0000000001300454000000060BB0653C000000020EDFF444000000020BADE52C0FFFFFF3F86FF44400000004086D852C000000060C7FF44400000008025D652C0000000C012FF4440000000C0C3D452C00000004008FB4440000000402ED252C0FFFFFF3F48F944400000004050D052C00000006034EF4440000000A0FCCA52C00000008012EF4440000000C083C952C0000000A087ED44400000004002C852C000000020AFEC44400000000090C752C0FFFFFF7F1FEB44400000008022C552C00000002033E84440000000C038C652C00000000008E64440000000203EC652C0FFFFFFBFB4E3444000000000F3C352C00000000095E24440000000C0ABC352C0000000E001DD4440FFFFFFDF35C452C0000000E07CDB4440000000404FC352C0000000A090D1444000000060A9C452C00000008076CE4440000000007FC452C0000000A05CCD444000000060A1C152C00000004069C8444000000040FAC052C0000000400DC544400000002038BE52C0000000C0E3BD444000000080B5BB52C000000040FCBD44400000006088B952C0000000601CBB4440000000A052B952C000000000E8B84440FFFFFF3F53B752C00000006039B94440FFFFFF9FC3B252C00000008004B744400000008055B052C0000000000CB74440FFFFFF3F63AF52C0FFFFFFFF02B64440000000C069AF52C0000000C065B34440000000A029AD52C0000000E0FEAF444000000040D4AC52C000000080DCAC444000000000B1B252C000000060EBA74440000000A0D9B252C000000080C6A5444000000060D6B452C0FFFFFF9F2CA44440000000A079B752C0FFFFFF3F059D4440000000E03FB752C000000060749A4440000000E091BA52C0FFFFFF9F0C92444000000000D3BC52C0000000204E8E4440000000E009BF52C000000060B58C44400000002050BF52C000000000748A444000000000DEBD52C0FFFFFFFF918A44400000002019C052C000000060FC874440000000E047C252C00000000099834440000000C080C452C0000000005C814440000000E07CC752C00000000005804440FFFFFF7FF0C852C0000000801C7D4440000000A0B2C852C0000000603E7B4440000000A020C552C0000000C09D73444000000080BDC452C00000004038714440000000C0A3C352C0000000009C6F44400000002085C352C000000080836D44400000002065C652C0000000806A6B4440000000E0BBC552C000000020206944400000008075C652C0000000E051654440000000605DC852C0000000E0E4624440000000A0ECCA52C0000000C029634440000000E06ACC52C000000060BB5F4440FFFFFF5F0ACC52C000000020A45C4440FFFFFFBF2ACD52C0000000A0CD574440000000E0CDCB52C0FFFFFF7FB85544400000002065CD52C0000000204853444000000040B1CC52C0000000602A51444000000060D9CC52C0000000A0AC4E4440000000C068CC52C0FFFFFF9FB54A444000000040ABCC52C0000000E008494440000000C0B0CB52C0000000E04147444000000080FEC752C000000000484844400000008021C552C0000000E0CA454440000000A019C452C0000000E0AC4244400000002083C452C000000020633A4440000000A0B3C352C0000000C0C4354440000000C062C152C0000000205B334440000000200EC052C0000000404A3444400000006049BE52C000000080C133444000000040D6BC52C000000000352C444000000080B4BB52C0000000A0B52A4440FFFFFF7FFABA52C0000000C02E2844400000002063B852C0FFFFFF7F5526444000000040F6B552C0000000C0C91F4440000000604FAF52C000000020BC164440000000C074AE52C0000000E018134440000000C0C9AF52C000000060E70F44400000002015B552C0000000C0DA0E444000000000D4B752C0FFFFFF1FFA094440000000C038BD52C0FFFFFF5F69074440000000C0F7BE52C0000000E058044440FFFFFF9FF2C252C000000080F60044400000002061C452C0000000A01DFE43400000002070C552C000000020E1FC43400000004020C752C00000008000FD4340000000C0F9C852C0000000E057FA43400000004071C952C0000000E0A1F7434000000040B7C852C000000080C9F24340000000802BC952C0000000A0D4F04340FFFFFF1FE7CB52C0000000204BF04340000000E0D4CF52C000000060D6EC43400000006043D052C0000000E036EC4340000000C029D652C0FFFFFF7F98EC4340000000E0EEDA52C00000006041E64340000000201ADE52C0FFFFFF9FC8E94340000000E05CE552C00000006085EB4340000000E03CE952C0000000004AEB4340FFFFFFBF7CEC52C00000002001E9434000000060BDEF52C0000000402DE34340000000009EF152C000000080BADC4340000000E0A6F252C000000000A4DC434000000060EE0853C0000000806EDC4340000000C0F00E53C00000004062DC4340000000407D2453C0000000402EDC4340000000609C3253C0000000C04EDC4340000000A0D03F53C0000000E042DC4340000000802A4E53C0000000003CDC434000000000BE5D53C00000004028DC434000000020785E53C0FFFFFF7F19DC4340 Pennsylvania 42 0.67013888888888888889 0.68792270531400966184 0.68104426787741203178 0.66420118343195266272 0.66613418530351437700 0.70882352941176470588 0.71606648199445983380 0.69239631336405529954 0.67017913593256059009 0.70817610062893081761 0.66852057842046718576 0.63291139240506329114 0.66666666666666666667 0.61039794608472400513 0.71939736346516007533 0.78173858661663539712 0.77858880778588807786 0.73363688104723961298 0.77062146892655367232 0.79074889867841409692 0.77839335180055401662 0.74795180722891566265 0.77476255088195386703 0.74634960367125573634 0.77400899060073559461 0.75652173913043478261 0.75781559161060546102 0.75045471080392870135 0.77366548042704626335 0.79238374221896741120 0.79096045197740112994 0.78639781271360218729 0.76725838264299802761 0.76601758793969849246 0.77261538461538461538 0.78271096736254042929 0.78984091543399385989 0.78471863706763035622 0.77377830750893921335 0.78933153274814314652 0.78708479471838250464 0.80098231827111984283 0.81018867924528301887 0.82201158504476040021 0.82807242429097900977 0.83722295611331278438 0.85232766956761983699 0.86239695624603677869 0.86007805325987144169 0.84918200408997955010 0.84085315832649712879 0.81606238443604791382 0.80663541291020555355 0.79762463933436220895 0.78834103424267358694 0.76860068259385665529 0.76986622608324894740 0.75786006587483780816 0.73724594656314226992 0.72158120548173449411 0.72585959607706322441 0.74143476959904248953 0.76331757435878345680 0.75257551946918107211 0.75032092426187419769 0.74885366173195336041 0.74304316524243277929 0.74205305927342256214 0.73300370828182941904 0.73048168322118978960 0.72786259541984732824 0.72684547244094488189 0.71158258236949785946 0.73391875490826015564 0.74788905137034765791 0.73471972272212224222 0.72585172897792522649 0.71138012633719906909 0.70467272390135360653 0.71537558685446009390 0.73625227548543689320 +43 0106000020E61000000100000001030000000100000082000000000000608B5052C0000000407C5F454000000020435152C0000000E0AA5F4540000000C0FB5252C0FFFFFF1FD866454000000000EC5152C0000000A0236B454000000060B15152C0000000005978454000000080085052C0000000A0C7A7454000000020494F52C0FFFFFF3FA2C14540000000E0AA5052C0000000A097C74540000000A0AD5252C00000008098C94540000000400F5252C000000060EBCB4540000000E0D95252C0000000A04ECF454000000060745352C000000080F4CF4540000000E04D5752C000000020B6CE454000000020DE5852C000000080D7C8454000000000CD5A52C0000000808CCA454000000000195B52C000000000E7D0454000000080C55752C0000000806BDB4540FFFFFFDFDA5652C0FFFFFFDFD4E0454000000000015752C000000080A1E34540000000C0A65852C0FFFFFF1FF8E6454000000020095852C0000000C065F1454000000020F85952C00000008016F54540000000E0BD5A52C0000000607BFE4540000000E0275A52C0000000405302464000000080EE5B52C000000000D7054640FFFFFF9FE15B52C0000000002C08464000000020305A52C0FFFFFF9FA30D464000000080215A52C0FFFFFF1F6E114640000000C07A5852C0000000E005164640000000402D5852C0000000C0C019464000000080915352C0FFFFFF9F4A21464000000040225552C0FFFFFFFF042F4640000000203A5352C0000000E0E6334640000000C0CC5252C00000002066374640000000A06E5552C000000000AB45464000000080495652C0FFFFFFFFE6464640FFFFFF3FCA5752C0FFFFFF9F204A4640000000C0765852C000000060544F464000000040B75752C0000000E030514640000000C0E75752C000000040A3544640FFFFFFDFF25652C0FFFFFFDF14574640000000C0E85752C000000060B25C464000000000F15452C0000000E04D66464000000080A55752C0FFFFFF7FD7684640000000A07E5852C0000000C0876C4640000000C08E5552C0000000405E774640000000C0795652C000000000B07D4640FFFFFFDF165652C0000000E0C780464000000020184C52C0000000C014814640000000000D2352C000000080AE804640000000C0BFF951C000000000EF804640000000C05FE051C000000000B481464000000040A6E251C0FFFFFF7FFE7C4640000000E01DE151C0FFFFFF7FC97846400000000070E051C0000000E02673464000000020D6E451C0000000C071684640000000C05FE551C000000020BB634640000000206CE851C0000000C0EE5E4640000000E0EBE651C000000080C2564640000000C0B5E551C00000002045534640FFFFFF3F62E451C00000002096514640000000207EE351C0000000405B4C46400000008062E251C0000000A0184A4640000000C0EFE551C0FFFFFFFF8B464640000000E0E1E551C0FFFFFFFFF744464000000080D8E451C0000000C04B434640000000E092E551C0000000204B3F4640FFFFFF1F57E751C0000000C0BA3C464000000000C5E851C0000000A0033D4640000000C07BE951C0000000000C3C4640000000200AEA51C0FFFFFF7F54384640000000C059EB51C0FFFFFF9FEC3546400000002017F151C0000000E0F7324640000000A015F351C0FFFFFF9F2A314640FFFFFF1F96F451C0000000A0D62C46400000004075F551C0000000C00C2C4640000000E071FB51C0000000C0032B4640000000A01EFC51C0FFFFFF5FB1294640FFFFFF5FACFF51C000000020EB294640000000204D0252C0000000E051264640FFFFFF7FD70352C0000000A076214640000000E0DE0252C000000020FE1D4640000000E0D20352C0000000804F17464000000060E40252C00000000004144640000000C0430252C0FFFFFF7F720F4640FFFFFFDF320352C000000080D90C4640000000401B0252C0000000E04A0C4640000000A0400252C0000000E0A90A4640000000E0F30452C0000000C017044640000000A07B0552C0FFFFFF3F2201464000000060100752C0000000E09CFE4540000000E03F0752C000000040FCFC454000000040E60552C0000000409DFA4540FFFFFF5F460752C00000006034F84540000000C0D00752C0FFFFFFFF5EF4454000000060EA0A52C0000000207EF0454000000000DC0B52C0000000809BE64540FFFFFF3F380D52C000000040DDE14540000000C00D0E52C00000006014E0454000000060AC1052C0000000001CDE4540000000007D1352C00000002067D94540000000E0581552C0FFFFFF1F74CC454000000000EF1752C00000004041C94540000000404F1952C0000000E03CC24540000000C0821852C00000000006BE4540000000C0631952C0000000A07DB44540FFFFFF1F681A52C00000004043B04540000000607A1952C000000060EBAC4540000000E0481A52C0FFFFFFDF62A94540000000E0C81952C0000000E055A7454000000080E81B52C000000000B89D4540000000C0FB1C52C000000000F693454000000060091C52C000000060DF8E4540FFFFFF5F691C52C0000000601B8A454000000000951D52C000000040F1854540000000C0491D52C000000060F07F4540000000E0521E52C0000000A0EF7C4540000000804D2052C0FFFFFF9F957B4540FFFFFFDF522152C0000000C0CD794540000000209E2152C0000000E0CD74454000000000732352C0000000E0266E454000000040852252C00000002061674540000000C0DD2052C0000000C003654540000000C07E2052C00000002063624540000000E0B41E52C00000000079614540000000E09B1D52C0000000C0955F454000000000331D52C000000000E65C4540000000E0193B52C0000000405F5E4540000000204A4152C0000000A0C25E4540000000608B5052C0000000407C5F4540 Vermont 50 0.55034722222222222222 0.55652173913043478261 0.53802497162315550511 0.53994082840236686391 0.53993610223642172524 0.56323529411764705882 0.57340720221606648199 0.54262672811059907834 0.51106427818756585880 0.57484276729559748428 0.54616240266963292547 0.50243427458617332035 0.55240549828178694158 0.49743260590500641849 0.58505963590709353421 0.59599749843652282677 0.62956204379562043796 0.62037564029595902106 0.63672316384180790960 0.65638766519823788546 0.62326869806094182825 0.56337349397590361446 0.60334690185436454093 0.57571964956195244055 0.58520637515324887617 0.60289855072463768116 0.60308666402849228334 0.60203710440160058203 0.61209964412811387900 0.63419992676675210546 0.64689265536723163842 0.65721120984278879016 0.65548980933596318212 0.65075376884422110553 0.65476923076923076923 0.66627462511026168774 0.68545911247557912364 0.70443985544656685596 0.68772348033373063170 0.70402880936304298897 0.69898906540127914174 0.71394891944990176817 0.72754716981132075472 0.73301737756714060032 0.72872937029322224003 0.71466314398943196830 0.71722613620665837823 0.72961318960050729233 0.70925160697887970615 0.71942740286298568507 0.71579618995533679701 0.69957392073317790819 0.70082942661377569419 0.69026370529423605985 0.69422115323754668017 0.68532423208191126280 0.69253317699728188456 0.69078750374288851183 0.68472253939255537794 0.67467821885283458991 0.67903244563321316432 0.67530670257330939557 0.67818188586531660648 0.67375589313776846517 0.66836700222957908249 0.67316258351893095768 0.66857607913106081948 0.66607911089866156788 0.65630969771884481402 0.66226102744846737157 0.65875318066157760814 0.66193405511811023622 0.66557865607038955510 0.68182575378977178078 0.69835841313269493844 0.69408340554111399942 0.67883118540257751691 0.67263802241214088749 0.66786575190061190432 0.67745011737089201878 0.69690533980582524272 +25 0106000020E610000001000000010300000001000000C4000000FFFFFF7F0B5A59C00000000021004440000000C0448359C0000000E0DCFF4340FFFFFF5F038359C000000000DC2B4440000000A0048359C0000000C02D374440000000A0EA8259C0000000E04159444000000060FA8259C0000000001F5F4440000000A0068359C000000000C17F444000000000BBA759C0FFFFFF5F0780444000000020B7A959C0000000E0C27F4440000000607AD859C0FFFFFFDF0A804440000000C098E459C000000000F57F4440000000E046035AC0000000C069804440000000A06C035AC000000080ADB14440FFFFFFFF84035AC0FFFFFF3F39C844400000002066035AC00000002060D944400000008064035AC000000020FBFF4440FFFFFFFF90035AC0000000A0AE4E4540000000C090035AC0FFFFFF5F658045400000002010E059C0FFFFFFBFD37F4540000000A058C059C000000080EB7F4540000000806DB259C0000000C0667F454000000080858559C0000000C0B47E4540FFFFFFBFCD4E59C0000000A0507E4540000000C0A70C59C000000000DC7E4540000000E012E258C0000000C0047F4540000000E03AD058C0000000A0067F4540000000A0D39F58C0000000C0F27E4540000000E0409D58C000000000F5774540000000A0039958C000000020C7754540FFFFFFBFD69358C0000000C0DE704540FFFFFFDFB78A58C000000020776B4540FFFFFFBF3F8958C000000060FB6A4540FFFFFF5FDB8758C000000020FD68454000000020C68758C0000000607867454000000040198258C0000000E074624540000000C0AA7F58C0000000E02662454000000040A57D58C00000004008634540FFFFFFDF767B58C0FFFFFFDF6A65454000000020EF7858C000000020676A4540FFFFFF1FDA7858C0000000208B6D4540FFFFFFFF5E7458C000000060EC6E4540000000E0FC7258C0000000A0BF6C4540FFFFFFDF657158C0000000204F6C4540000000E0646E58C000000040D36D4540FFFFFFBFDD6B58C0000000801D6B4540FFFFFFBF9D6858C0FFFFFF9F3E6E454000000000806458C0000000008B6C4540000000E05E6058C000000000196E454000000080E65E58C000000060B76D4540000000403E5D58C000000060DB6C4540000000E0E45858C000000020086F4540FFFFFFBFE85358C0000000A04E6E4540000000205A5158C000000060CD6C4540000000008B4F58C0FFFFFFBF086D4540000000E0574E58C0000000A0AC6B4540FFFFFF3F894D58C00000008002684540000000604F4A58C0000000403966454000000040544858C00000000010634540000000C0F24058C0000000C03861454000000060AC3E58C00000008010614540000000400F3E58C0000000C043604540FFFFFF1F903E58C000000080185D4540000000E01B3E58C0000000A04E5C4540000000401B3A58C0FFFFFF5FA85D4540FFFFFF1FD43358C0000000801F5A454000000000D93358C0000000403657454000000040233358C000000040C3554540000000E03A2E58C0000000809455454000000040B82C58C00000002030544540000000206F2C58C0000000C011524540000000E0C12D58C0FFFFFFBF9A4F454000000000AE2D58C000000000604E454000000020BA2858C0000000807E46454000000040412858C0000000C0E7424540FFFFFFDFBA2658C000000040ED40454000000080672558C0000000605742454000000080002358C0000000E09F42454000000020A41F58C000000000863E4540FFFFFFFF191C58C0000000A09F3E454000000040541958C000000000D43B454000000000721958C0000000E08C38454000000060B51A58C0000000A01735454000000040551A58C000000020C2304540000000A0201B58C000000060B52C454000000020ED1858C0FFFFFFFF152A4540000000C0931758C0000000E025264540000000C0EC1558C0000000801B24454000000040451558C0000000005221454000000000981558C000000040611D4540000000C03E1758C000000000661B4540000000E0841658C0FFFFFF5F87154540000000803A1258C0000000A0CD0F4540000000C0F81058C0000000A04206454000000060420F58C000000040A4034540FFFFFF3F170F58C0000000A02900454000000080F60C58C00000008091FF4440000000C0D50B58C00000008087FD444000000000690958C0000000A0AEFB444000000020510958C0000000C063F6444000000020380A58C0000000A0BBF3444000000040A90858C0FFFFFFBF6AEE444000000040DF0458C0000000404FE5444000000080560658C0000000E061E04440000000E05D0658C000000040A4DD444000000000750558C0000000603DDA444000000060CD0758C000000040F3D84440000000A0AD0758C0000000C090D7444000000040560658C0000000E0CCD34440000000E01A0758C000000080ACCC4440000000A0270558C0000000A0BAC9444000000080DD0558C00000006015C84440000000A0790558C0000000C0CDC4444000000040310358C0000000A01DC3444000000080460058C0000000A0B1C444400000006098FF57C00000002099C34440FFFFFFFFC4FF57C0000000A079C14440FFFFFF9FD70058C0000000A01ABF4440000000406C0058C0000000E0B0BD444000000040FCFC57C00000006077BC444000000060D3FB57C0000000802FBB4440FFFFFF1F25FC57C00000002089B24440FFFFFF9F53FC57C0000000C087AB444000000060E2F857C00000000094A64440000000606DF957C000000000B8A44440FFFFFF5F4CFA57C0000000007CA744400000002084FB57C0000000E0A9A64440000000C048FA57C0000000E0D49C44400000006001FB57C0000000009B9A4440000000A09CFA57C000000020D798444000000060F8F657C0000000E01B9744400000004002F757C0000000E05B954440FFFFFFDF16F857C0000000A00495444000000040E9F657C0000000E0F98D4440000000A039F857C0FFFFFF7F6E88444000000000FEF657C0000000007B8444400000004014F757C0000000E056804440000000A096F557C000000080B47C44400000004085F557C0FFFFFF7F577344400000002062F557C000000000666F44400000006027F657C000000020966C4440000000207FF657C0000000E073654440FFFFFFDF15F857C0FFFFFFDF7E5D44400000006022F157C0000000A0515244400000002077F057C0FFFFFFBF794F4440000000E019F157C0000000E0654B444000000040D7F057C0000000C05C4644400000002027EF57C0FFFFFFBF244444400000006046EC57C0000000A01643444000000020FAEB57C000000060D4474440000000203AEB57C0000000406D4844400000004069EA57C000000060844744400000004019EA57C0000000E0E143444000000020D2EB57C0000000E08F414440000000607CEC57C0FFFFFF7F1F3E4440FFFFFF3FBDE857C0000000E0BC3244400000002092E857C000000020ED2D4440FFFFFF7F6BE757C0000000005A2C4440000000E087E757C0000000E06B2A4440FFFFFF5F4CE957C0FFFFFF9F422944400000004061E957C0000000E090274440000000E018E657C0000000C0A62744400000000000E357C000000060132244400000000080DE57C0000000A0091D444000000020D9DD57C0000000004C1B4440FFFFFFFF7BDD57C00000008045164440FFFFFF9F05DB57C000000000DD104440000000A01FD957C0FFFFFFFFC50E4440FFFFFF1F98D857C0FFFFFFDF340C444000000060D3D957C0000000E0490A4440FFFFFFDF76DA57C0000000802806444000000040FAD857C0000000A099054440FFFFFF3FBED757C000000020AE0344400000006011D657C0000000603203444000000080BDD357C0FFFFFF9FECFF4340000000A015D557C0000000600DFF4340FFFFFF9FF2F157C0000000A02AFF434000000000100058C0FFFFFF5F61FF434000000040610F58C0FFFFFFDF4BFF4340000000200A1D58C00000000041FF434000000080453358C0000000E04AFF4340FFFFFF5F1C3A58C0000000E081FF434000000060245758C0FFFFFFFFA9FF4340000000C03D7458C0000000C0F6FF434000000020797B58C0000000E0CCFF4340000000C0E29058C000000040CCFF4340000000E043A058C000000080A1FF43400000002019AE58C000000020CDFF4340FFFFFFDF1EC458C000000020C9FF4340000000A061CB58C0000000C0F1FF43400000008028E858C0000000A061004440000000408D0B59C0000000800F00444000000060340C59C0000000001300444000000000042F59C0000000C0E4FF434000000080483059C0FFFFFF5F06004440000000E0965459C0000000C03B004440FFFFFF7F0B5A59C00000000021004440 Nebraska 31 0.51736111111111111111 0.50338164251207729469 0.46878547105561861521 0.45414201183431952663 0.43929712460063897764 0.38088235294117647059 0.56648199445983379501 0.45622119815668202765 0.43730242360379346681 0.50943396226415094340 0.44493882091212458287 0.43037974683544303797 0.47336769759450171821 0.52759948652118100128 0.63967357187696170747 0.68230143839899937461 0.72141119221411192214 0.67501422879908935686 0.71977401129943502825 0.85792951541850220264 0.74570637119113573407 0.75180722891566265060 0.74219810040705563094 0.73466833541927409262 0.68492031058438904781 0.72587991718426501035 0.65294815987336762960 0.60931247726445980356 0.68896797153024911032 0.74148663493225924570 0.71398305084745762712 0.72624743677375256323 0.70151216305062458909 0.71984924623115577889 0.72030769230769230769 0.70332255219053219641 0.74127825844264582752 0.74599896747547754259 0.71275327771156138260 0.71280666216520369120 0.73695069114916443161 0.74577603143418467583 0.77754716981132075472 0.79462875197472353870 0.84425572824867809646 0.80214296198444150888 0.85205138831330294240 0.81838934686112872543 0.80268595041322314050 0.83026584867075664622 0.80065627563576702215 0.74539753999517646113 0.77064551027767760548 0.75340535462658525129 0.73428698018861953288 0.73765642775881683732 0.73245216649789479294 0.70940213594171074958 0.68668645809545558347 0.66580580663973007873 0.65426212350273287592 0.67654099341711549970 0.69858169229051111194 0.68755020080321285141 0.68127153570704682116 0.69330538451460762479 0.69477572229004288353 0.71836161567877629063 0.69080795595010675357 0.69051052013243618498 0.68826972010178117048 0.68476870078740157480 0.68823765935807374820 0.70200613978724923253 0.73734610123119015048 0.71921171321291297296 0.71019948109395602059 0.67965892867619736765 0.68384943445206749490 0.69923708920187793427 0.70268886529126213592 +17 0106000020E6100000020000000103000000010000008F00000000000040C77151C0000000407B09464000000060117751C000000080FDFF4540FFFFFFBFB07251C0000000A0C4E04540000000802D7551C0FFFFFFDF2BDD4540000000008C7651C0FFFFFF5F43DF4540000000C02F7651C000000040CFEB454000000080C97851C00000008035F0454000000040D57951C00000000034E54540000000604C7E51C0000000E066E2454000000020008051C0FFFFFFBF9FE4454000000060397F51C0000000803EEC4540000000E0B88151C0000000003AEC4540000000800E8A51C0FFFFFFDF15E54540000000A01F8F51C000000080C5D7454000000080418E51C000000040E0C9454000000040E59551C00000002075C44540FFFFFF9F739751C00000006011B7454000000060479D51C0000000A0B8AC45400000004086A251C000000000F6AA4540000000A0A2AA51C0000000A0A48B4540000000406DB451C0000000A0968F4540FFFFFFDF2FB551C0000000005D944540FFFFFFBF13B451C000000000199E4540FFFFFF7FB3B951C0000000C0F5A34540000000C000BA51C000000080A7A64540000000A017BE51C0FFFFFFDFE2AE4540000000A0B1BE51C000000080B3B245400000000091BD51C0000000E011B845400000008029BE51C0000000402DBC4540000000E06CBD51C00000006016C24540000000A0BEBD51C0000000A015C44540000000A0CEBC51C00000008041C64540000000C03FBD51C00000004033C84540000000005CBE51C0000000202FC945400000002009BF51C00000006042E54540FFFFFFBF94C051C0000000401B24464000000080DEC151C0000000E090554640000000A0A1C551C00000000095A64640000000606EBD51C0FFFFFF5F5EAB4640000000A01FB851C0000000C0D99C4640000000A0F9B551C00000006098A34640000000A00AB451C00000008064AD46400000006018B551C0000000C001B246400000006009B351C0000000806AB64640000000A0AAA851C0000000602AB24640000000E01AAE51C000000000A7C146400000004060A351C0000000208FD4464000000040669951C0000000806ADC464000000020AB9A51C0FFFFFF5F27E54640000000E0489051C00000004011F3464000000060DE8F51C000000000E8F84640000000C0E39351C0000000C0FFFB4640000000C0F39151C000000080CC064740000000808A9351C0000000008708474000000020B58E51C00000004096114740000000A02C9251C0000000E05818474000000020428C51C0000000E0DA2A474000000080038351C0000000E089364740000000A0EF8051C00000004008494740000000C0117F51C0000000807D58474000000040C54E51C0000000A005BA4740000000E0094351C0000000E003B6474000000000624251C0FFFFFF1FF0A04740FFFFFFFF4D3951C00000000053974740FFFFFFBFF82051C0000000A001A64740000000C0121951C0000000C07CA4474000000000761551C000000020BDAD474000000000CE0E51C0000000E011AD4740000000C0A8F250C000000080CD87474000000020F9F150C00000008037F94640000000E064F050C0000000A050F5464000000020E3F250C0FFFFFFFF6FF0464000000060A2F050C000000060F3E94640FFFFFF1F6EF350C0FFFFFF7FB0E546400000006074F350C000000060CAD646400000006039F050C0FFFFFF7F61D4464000000040FDED50C0000000C032D746400000008067E750C00000000075CD4640000000A026DC50C0000000E0D6CB464000000040AADA50C0FFFFFF3F72C04640000000604CE050C0000000002DBE4640FFFFFFBFD2DA50C0000000A019B04640000000E09FDE50C0000000E0DDA34640000000E028DC50C0000000E041984640000000A027D650C000000080A38F46400000000094D150C0000000006397464000000080A7CA50C000000000FE934640000000C0ADC950C0000000E09A8F4640FFFFFF3F38C450C0FFFFFF9FC77A4640000000006DC950C0000000C0C67346400000000012BE50C0000000C00E6A4640000000E087C050C0000000E0E863464000000020DCCC50C000000080AC53464000000040C7D350C000000000A4534640000000A0E6D850C0000000407D5846400000006094E450C0000000A0934C464000000040A4E750C0000000002445464000000020F4F350C000000040E7464640000000C0FBF650C0000000A09B444640FFFFFF5FA3F950C0000000A0E53946400000006002FE50C0000000A04E3C464000000000B2FD50C000000000AC404640000000402CFF50C0000000C00B3E4640000000A0150151C0FFFFFF9F43314640000000A0CB0451C0FFFFFF3FCE30464000000080C10851C000000000D23C464000000000C10F51C000000000CB3E4640000000C0501751C0000000203537464000000080761B51C0000000A08C3B4640000000C05F2351C0000000801133464000000080F52151C0000000C016254640FFFFFF5FD62351C00000006041214640000000E0692F51C000000000522C4640000000600E3451C0000000C0E6294640000000601D3451C0FFFFFF1FFB344640000000E07A2F51C000000040EC40464000000040BB2F51C0FFFFFFFFAF464640000000A0BD3451C000000040EE4D4640000000E0C13451C000000080FE544640000000C01C3751C0000000E0314E464000000040BD3351C000000000E8484640FFFFFF1FFB3351C0000000604C3F4640FFFFFFBF6B3D51C0FFFFFF9F1237464000000040133F51C000000040B122464000000080684151C0000000E03B1F464000000060CC4451C000000080D4084640000000C00E4E51C0000000A02DF94540FFFFFF7FD35251C00000000097F84540FFFFFF9F345651C0000000A008024640000000A0475951C0000000C034034640000000A0F55E51C0000000C08BF14540000000E0BF6551C0FFFFFFBF21EC4540000000C08E6A51C00000000013ED4540000000E0F76951C00000004076FD4540000000A0426751C0000000E04A04464000000040276E51C0000000200DF8454000000040F06F51C00000008057F2454000000060696E51C0FFFFFFFF66E44540000000400E7051C0000000C07CE14540FFFFFFBFCD7151C0000000A045E54540000000C03B7351C0000000E06D034640000000E01A7151C0000000A01906464000000040C77151C0000000407B0946400103000000010000000900000000000080DC1851C0000000404730464000000060731651C0FFFFFF3F0E33464000000080C81651C000000040E2364640FFFFFFDF4F0F51C0FFFFFF7FFF37464000000080940A51C000000020CE2A464000000020891351C0000000201D254640FFFFFF5F8F1451C0000000C0CC1C4640000000C0D11951C000000000A722464000000080DC1851C00000004047304640 Maine 23 0.52170138888888888889 0.55652173913043478261 0.55732122587968217934 0.55769230769230769231 0.59265175718849840256 0.61176470588235294118 0.59556786703601108033 0.58294930875576036866 0.53740779768177028451 0.59245283018867924528 0.55061179087875417130 0.51217137293086660175 0.54209621993127147766 0.55006418485237483954 0.69177652228499686127 0.68918073796122576610 0.65632603406326034063 0.64541832669322709163 0.66045197740112994350 0.68281938325991189427 0.65540166204986149584 0.57590361445783132530 0.59656264133876074175 0.59991656236962870254 0.59215365753984470781 0.59958592132505175983 0.62564305500593589236 0.60712986540560203710 0.61209964412811387900 0.65800073233247894544 0.65324858757062146893 0.65584415584415584416 0.62557527942143326759 0.62280150753768844221 0.63046153846153846154 0.65039694207586004116 0.66508512419759977672 0.65539494062983995870 0.63361144219308700834 0.64641008327706504614 0.64782339591499896843 0.67249508840864440079 0.67811320754716981132 0.67825171142706687730 0.69203653260695401378 0.69925143108762659621 0.69332780770824699544 0.72390615091946734306 0.70500459136822773186 0.69028629856850715746 0.68334700574241181296 0.67593858027172602299 0.66577713667508113956 0.66248406361135341877 0.66782707766314323691 0.66353811149032992036 0.66796354527527580877 0.67187344046312007186 0.66659054578670929436 0.65868288415878702045 0.65457223708183122068 0.65376271693596648713 0.65748427204705356810 0.64082416623013794308 0.63543003851091142490 0.63968950609196908162 0.63354931605471562275 0.63614364244741873805 0.62661535003933026183 0.62824415251521948094 0.62603053435114503817 0.63048720472440944882 0.64022327869627947681 0.65993193879250850765 0.67755554507288079626 0.67099913350663200693 0.65333248266768746544 0.63246826902391801772 0.62340070461709623586 0.64014818075117370892 0.66876516990291262136 +13 0106000020E610000001000000010300000001000000F8000000000000E0ADC756C0000000A04A5A44400000002044C856C0000000A050574440FFFFFF5F66CA56C00000000002544440FFFFFF1FC1CD56C0000000A068524440000000A0C5D056C0000000A0DC514440000000000AD856C0000000803D4D4440000000C04FDA56C00000002057494440000000806CDA56C0FFFFFF9F244644400000006074D856C0000000C0A5434440000000A0FCD756C0000000C07740444000000060AAD856C0FFFFFF7F3F39444000000040DBD756C00000002095334440FFFFFF3FB0D856C0000000E038324440000000E0CDDA56C0FFFFFF1F85314440000000C0B5DC56C0000000809A2F44400000004085DE56C0000000400B324440000000C05EDF56C00000008004324440000000A003E056C0000000E0DA33444000000020C3E156C0000000007F34444000000080E2E156C000000080AF374440000000E079E256C0000000C07A384440000000801DE256C0000000804A3A44400000000012E556C0000000205B3B4440000000E07EE556C000000060033E44400000000078E756C0000000609E40444000000000D5E756C0000000E0344444400000006048EC56C0FFFFFF1F9C464440000000A025EC56C0000000A0634A444000000040E0ED56C000000080F44B44400000008075EF56C0000000400C4E4440000000808EFC56C0000000A0DA4D4440000000005A0C57C0000000C0CE4C444000000000201757C0FFFFFFFFBD4C4440000000005C2957C000000040B44B444000000080ED2D57C000000080794B444000000060724657C0FFFFFF7FCB4A444000000000AF5757C0000000404D4A444000000020036457C0FFFFFFDF574A4440000000204F7257C0000000800A4A444000000020248157C0000000A079494440000000003E8F57C0000000A015494440000000200A9F57C0000000E07F494440000000C0EFA857C000000060B249444000000040E7BA57C0000000A0E249444000000020E6CD57C0000000A07B4A44400000008077D857C0FFFFFF7FCB4A4440000000E019F157C0000000E0654B44400000002077F057C0FFFFFFBF794F44400000006022F157C0000000A051524440FFFFFFDF15F857C0FFFFFFDF7E5D4440000000207FF657C0000000E0736544400000006027F657C000000020966C44400000002062F557C000000000666F44400000004085F557C0FFFFFF7F57734440000000A096F557C000000080B47C44400000004014F757C0000000E05680444000000000FEF657C0000000007B844440000000A039F857C0FFFFFF7F6E88444000000040E9F657C0000000E0F98D4440FFFFFFDF16F857C0000000A0049544400000004002F757C0000000E05B95444000000060F8F657C0000000E01B974440000000A09CFA57C000000020D79844400000006001FB57C0000000009B9A4440000000C048FA57C0000000E0D49C44400000002084FB57C0000000E0A9A64440FFFFFF5F4CFA57C0000000007CA74440000000606DF957C000000000B8A4444000000060E2F857C00000000094A64440FFFFFF9F53FC57C0000000C087AB4440FFFFFF1F25FC57C00000002089B2444000000060D3FB57C0000000802FBB444000000040FCFC57C00000006077BC4440000000406C0058C0000000E0B0BD4440FFFFFF9FD70058C0000000A01ABF4440FFFFFFFFC4FF57C0000000A079C144400000006098FF57C00000002099C3444000000080460058C0000000A0B1C4444000000040310358C0000000A01DC34440000000A0790558C0000000C0CDC4444000000080DD0558C00000006015C84440000000A0270558C0000000A0BAC94440000000E01A0758C000000080ACCC444000000040560658C0000000E0CCD34440000000A0AD0758C0000000C090D7444000000060CD0758C000000040F3D8444000000000750558C0000000603DDA4440000000E05D0658C000000040A4DD444000000080560658C0000000E061E0444000000040DF0458C0000000404FE5444000000040A90858C0FFFFFFBF6AEE444000000020380A58C0000000A0BBF3444000000020510958C0000000C063F6444000000000690958C0000000A0AEFB4440000000C0D50B58C00000008087FD444000000080F60C58C00000008091FF4440FFFFFF3F170F58C0000000A02900454000000060420F58C000000040A4034540000000C0F81058C0000000A042064540000000803A1258C0000000A0CD0F4540000000E0841658C0FFFFFF5F87154540000000C03E1758C000000000661B454000000000981558C000000040611D454000000040451558C00000000052214540000000C0EC1558C0000000801B244540000000C0931758C0000000E02526454000000020ED1858C0FFFFFFFF152A4540000000A0201B58C000000060B52C454000000040551A58C000000020C230454000000060B51A58C0000000A01735454000000000721958C0000000E08C38454000000040541958C000000000D43B4540FFFFFFFF191C58C0000000A09F3E454000000040B71E58C00000008031424540FFFFFF3F4C1F58C00000004032484540000000600A2058C00000004075494540FFFFFF7F3E1F58C0000000604D4A454000000060CD2058C0000000009C504540000000609D2258C0000000E0C9544540000000C0032458C0000000009255454000000020142858C000000080AB5A454000000040FC2858C000000060D25F4540000000A07D2858C0000000806F634540000000A06F2658C0FFFFFF1F58664540000000E0962558C000000000EE6A4540FFFFFFFFA82458C000000020CC6A4540FFFFFFDF932358C0000000A05F6C454000000080612258C000000020CE72454000000020D02258C0000000E0F9744540000000A0EF2058C000000000E8794540000000E0132158C0000000A0447E4540FFFFFFDFEA1F58C0000000408B814540000000C0422158C0000000409886454000000040AC1E58C000000040EC874540000000E08D1D58C000000020AD894540FFFFFFBF781D58C0000000E03F8B454000000060E01C58C0000000602B90454000000060421E58C0000000C0C39A4540000000E0291F58C000000000E59B4540FFFFFFFFBA2358C000000060DD9C454000000060442458C0FFFFFFDFAC9E4540000000C0CA2358C0FFFFFF7F6BA04540FFFFFF7F812458C0FFFFFF9FBEA14540000000400B2558C0000000C021A5454000000060932258C00000000062A74540000000E0712158C0000000C0B1AD454000000040952158C0000000002FB14540FFFFFF3FAC2358C0000000C04BB34540000000C0AE2558C0FFFFFF7FC0B74540000000A0572558C000000060B0BD454000000080452658C0000000E0FBBF4540000000E0721D58C0000000A0F7BF454000000020E30358C000000000D1BF4540000000A076F757C000000080DEBF454000000040BADD57C000000020F2BF4540000000A05CD957C0FFFFFF1F0CC0454000000080E4BA57C0FFFFFFBFECBF4540FFFFFF3F03B757C00000006002C04540000000801E9D57C000000040C3BF454000000060C78F57C0FFFFFFFFDEBF454000000060517E57C0000000400BC04540000000A0D26957C0000000801AC04540000000200A6057C00000008011C04540000000C0774357C00000000031C04540000000A0BA4157C0000000202BC0454000000080B32357C0000000A009C04540FFFFFFDFFD1C57C000000080EFBF4540000000A0F30457C0FFFFFF5FE5BF4540000000E0BBEE56C000000000F3BF4540000000E019E756C00000008015C04540000000C04CCE56C0000000601BC04540000000E016CF56C0000000A07BBB4540000000807DCD56C00000002048B64540000000E0ADCC56C0FFFFFFBF6DAF4540000000A052CB56C0FFFFFFDF4EAD45400000000004C556C0000000E01AA84540000000403EC456C0FFFFFF1FEEA345400000004069C456C00000008003A145400000008051CA56C000000060E4924540000000C0C7CA56C0000000A09C8A45400000004037CA56C0FFFFFFBF648A4540FFFFFFBFBBC956C0000000A02B80454000000040E5C856C00000002084764540FFFFFF9FF8C556C0FFFFFFBF8B6F4540000000E03DC556C0000000A045644540000000003AC456C0FFFFFF9F595F4540FFFFFF7FF0BF56C000000020815A454000000080D5BA56C0FFFFFF9F20574540FFFFFF5F1DB956C0000000C0D056454000000000B6AF56C0FFFFFFBF185445400000006075AC56C0000000C0A75145400000002083AA56C00000006023494540000000E0E6A856C0FFFFFF9F214745400000008009A856C0000000E0A743454000000060DAA856C0000000C032414540FFFFFF9FB6A956C000000040523F4540000000807EA956C0000000E0E13C4540000000E0C5A656C0000000A0F33A4540000000C011A456C0000000C0FE354540000000406D9F56C0000000A0C331454000000020439C56C0000000E0162E4540000000205F9B56C0000000E0992B454000000040C09A56C0FFFFFF3FC8214540000000200F9A56C0000000000F1F4540000000E0889756C000000020E81A4540000000E0B59456C0000000C041194540000000A0C78E56C0000000C071144540000000C0428C56C000000040B40F454000000000458B56C0FFFFFF9F6C0F454000000060AA8A56C000000080470D454000000020C28A56C000000040D0074540FFFFFF5FA28956C0000000604704454000000080218956C000000080F2FD4440000000A0E38956C0000000A023F74440000000A0888C56C0000000802FE7444000000000579056C00000000010E4444000000040839356C0000000E0D3E04440FFFFFF9FDD9456C0000000A082DC4440FFFFFF1FD59556C00000006015D34440FFFFFFDFB79556C00000008028CD4440000000A04B9656C0000000E01DCB4440FFFFFF7F129B56C0000000609CC84440FFFFFF7FD69B56C00000000094C54440000000A01E9D56C0000000A086C34440000000209DA256C00000000053C34440000000E071A656C0000000203AC14440000000A029AA56C0000000402DBB44400000006053AD56C0000000A09BB94440000000E0E9B156C0000000C093B944400000006006B656C000000060E9B8444000000020C7BC56C000000000EBB54440000000600BC056C0000000C02DB7444000000060C2C156C00000008035B644400000000092C356C00000006060B34440000000A0B0C456C0000000E0DDAA4440000000E08CC656C0000000E047A24440000000607FC656C000000080A29D4440000000C09AC356C0000000A08F964440000000202BC156C0000000C039954440FFFFFFBF61BF56C0000000C07A924440FFFFFF5F4CBD56C0000000A05B8D44400000000019BD56C0000000A001894440000000407CBD56C000000020AA79444000000000EEBE56C00000004043764440FFFFFF3F26C356C00000004096704440000000A0B0C556C0000000A0B76A4440000000A0EFC556C0000000607A614440000000E0ADC756C0000000A04A5A4440 Iowa 19 0.50434027777777777778 0.49275362318840579710 0.45402951191827468785 0.43934911242603550296 0.40415335463258785942 0.39558823529411764706 0.58864265927977839335 0.45276497695852534562 0.55110642781875658588 0.57610062893081761006 0.52836484983314794216 0.48782862706913339825 0.52319587628865979381 0.53594351732991014121 0.64281230382925298180 0.62726704190118824265 0.66423357664233576642 0.70859419464997154240 0.68757062146892655367 0.90418502202643171806 0.75124653739612188366 0.73831325301204819277 0.74084124830393487110 0.71964956195244055069 0.67715570085819370658 0.74037267080745341615 0.66086268302334784329 0.63950527464532557294 0.69003558718861209964 0.72867081655071402417 0.71539548022598870056 0.70437457279562542720 0.71663379355687047995 0.71733668341708542714 0.74830769230769230769 0.74948544545721846516 0.79067820262349986045 0.79168817759421786267 0.73730631704410011919 0.74544226873733963538 0.75345574582215803590 0.75874263261296660118 0.75566037735849056604 0.78515007898894154818 0.86492549270950168242 0.82137090855717011595 0.85536676336510567758 0.83449587824984147115 0.83597337006427915519 0.86278118609406952965 0.83073557560842220399 0.77747407347857544819 0.79105661738189686260 0.75333825404281017245 0.72694474333818596114 0.72798634812286689420 0.71390502584874487022 0.69967062581095917756 0.68047499429093400320 0.63793893447744407881 0.64201263712834825755 0.65005984440454817475 0.66481777910136619142 0.66139339968569931902 0.63945003715965137491 0.67136119481200052404 0.66300435095627132438 0.67856716061185468451 0.66855826497359253849 0.66335576204208052974 0.65178117048346055980 0.65755413385826771654 0.64606542255020222806 0.67185454891601818139 0.68010283503938865041 0.68935101868515185852 0.67321679213984943218 0.64883734574540903135 0.64913777118486927499 0.67352552816901408451 0.68232327063106796117 +39 0106000020E61000000100000001030000000100000091000000000000806DB259C0000000C0667F4540000000A058C059C000000080EB7F45400000002010E059C0FFFFFFBFD37F4540000000C090035AC0FFFFFF5F6580454000000040C1035AC00000002055BD4540FFFFFFDFAC035AC0FFFFFF7F7AC0454000000080C6035AC0000000C02CED4540000000A0CA035AC000000000AB124640FFFFFFFFDF035AC0FFFFFF9F46174640FFFFFF3FC6035AC0000000E08449464000000060CC035AC000000020A97F4640FFFFFF9FB9025AC0FFFFFF7FB87F464000000060C6025AC0000000E03F9B46400000004023035AC0000000E007F146400000004019035AC000000000B4F84640000000E0A3BF59C00000006076F84640000000808ABC59C00000006088F8464000000080268059C000000000A4F84640FFFFFF5FE52059C0000000C05EF846400000006006F858C0FFFFFF9FC6F84640FFFFFF7FE2ED58C000000000ADF84640000000A069C058C00000004044F84640FFFFFF3FB9AE58C0000000201AF84640000000C0EA8058C0000000803CF74640FFFFFF1F9D7E58C00000006026F7464000000080E84E58C0000000A0E0F74640FFFFFFBF422458C00000004092F74640000000609B2558C0000000C0B0E8464000000040AC2658C00000008076E74640000000000D2A58C0FFFFFFDF97DE4640000000C0463558C0FFFFFFFF4AD3464000000060B23658C000000000F9CD464000000060EF3558C0000000C0C4CA4640FFFFFF9F353158C0000000003EC2464000000040362F58C000000060A7BA464000000040572C58C00000000091B44640FFFFFF1FB42658C000000080C2B24640FFFFFFBF0F2258C00000008005B04640FFFFFF5F8B1E58C0000000C00DAA4640FFFFFFDF431D58C0000000E041A6464000000000111D58C0000000C03AA34640000000002B1D58C0000000806B7C4640000000E01C1D58C0000000C09366464000000080351D58C0FFFFFFFF7D504640000000201B1D58C0000000A0E9444640000000C0401D58C0000000E07B194640000000C0331D58C000000080A4EC4540000000E0721D58C0000000A0F7BF454000000080452658C0000000E0FBBF4540000000A0572558C000000060B0BD4540000000C0AE2558C0FFFFFF7FC0B74540FFFFFF3FAC2358C0000000C04BB3454000000040952158C0000000002FB14540000000E0712158C0000000C0B1AD454000000060932258C00000000062A74540000000400B2558C0000000C021A54540FFFFFF7F812458C0FFFFFF9FBEA14540000000C0CA2358C0FFFFFF7F6BA0454000000060442458C0FFFFFFDFAC9E4540FFFFFFFFBA2358C000000060DD9C4540000000E0291F58C000000000E59B454000000060421E58C0000000C0C39A454000000060E01C58C0000000602B904540FFFFFFBF781D58C0000000E03F8B4540000000E08D1D58C000000020AD89454000000040AC1E58C000000040EC874540000000C0422158C00000004098864540FFFFFFDFEA1F58C0000000408B814540000000E0132158C0000000A0447E4540000000A0EF2058C000000000E879454000000020D02258C0000000E0F974454000000080612258C000000020CE724540FFFFFFDF932358C0000000A05F6C4540FFFFFFFFA82458C000000020CC6A4540000000E0962558C000000000EE6A4540000000A06F2658C0FFFFFF1F58664540000000A07D2858C0000000806F63454000000040FC2858C000000060D25F454000000020142858C000000080AB5A4540000000C0032458C00000000092554540000000609D2258C0000000E0C954454000000060CD2058C0000000009C504540FFFFFF7F3E1F58C0000000604D4A4540000000600A2058C00000004075494540FFFFFF3F4C1F58C0000000403248454000000040B71E58C00000008031424540FFFFFFFF191C58C0000000A09F3E454000000020A41F58C000000000863E454000000080002358C0000000E09F42454000000080672558C00000006057424540FFFFFFDFBA2658C000000040ED40454000000040412858C0000000C0E742454000000020BA2858C0000000807E46454000000000AE2D58C000000000604E4540000000E0C12D58C0FFFFFFBF9A4F4540000000206F2C58C0000000C01152454000000040B82C58C00000002030544540000000E03A2E58C0000000809455454000000040233358C000000040C355454000000000D93358C00000004036574540FFFFFF1FD43358C0000000801F5A4540000000401B3A58C0FFFFFF5FA85D4540000000E01B3E58C0000000A04E5C4540FFFFFF1F903E58C000000080185D4540000000400F3E58C0000000C04360454000000060AC3E58C00000008010614540000000C0F24058C0000000C03861454000000040544858C00000000010634540000000604F4A58C00000004039664540FFFFFF3F894D58C00000008002684540000000E0574E58C0000000A0AC6B4540000000008B4F58C0FFFFFFBF086D4540000000205A5158C000000060CD6C4540FFFFFFBFE85358C0000000A04E6E4540000000E0E45858C000000020086F4540000000403E5D58C000000060DB6C454000000080E65E58C000000060B76D4540000000E05E6058C000000000196E454000000000806458C0000000008B6C4540FFFFFFBF9D6858C0FFFFFF9F3E6E4540FFFFFFBFDD6B58C0000000801D6B4540000000E0646E58C000000040D36D4540FFFFFFDF657158C0000000204F6C4540000000E0FC7258C0000000A0BF6C4540FFFFFFFF5E7458C000000060EC6E4540FFFFFF1FDA7858C0000000208B6D454000000020EF7858C000000020676A4540FFFFFFDF767B58C0FFFFFFDF6A65454000000040A57D58C00000004008634540000000C0AA7F58C0000000E02662454000000040198258C0000000E07462454000000020C68758C00000006078674540FFFFFF5FDB8758C000000020FD684540FFFFFFBF3F8958C000000060FB6A4540FFFFFFDFB78A58C000000020776B4540FFFFFFBFD69358C0000000C0DE704540000000A0039958C000000020C7754540000000E0409D58C000000000F5774540000000A0D39F58C0000000C0F27E4540000000E03AD058C0000000A0067F4540000000E012E258C0000000C0047F4540000000C0A70C59C000000000DC7E4540FFFFFFBFCD4E59C0000000A0507E454000000080858559C0000000C0B47E4540000000806DB259C0000000C0667F4540 South Dakota 46 0.36979166666666666667 0.35362318840579710145 0.27355278093076049943 0.27958579881656804734 0.20607028753993610224 0.27058823529411764706 0.42797783933518005540 0.28110599078341013825 0.34035827186512118019 0.40251572327044025157 0.38375973303670745273 0.35150925024342745862 0.40807560137457044674 0.48587933247753530167 0.53107344632768361582 0.61225766103814884303 0.66058394160583941606 0.64428002276607854297 0.71751412429378531073 0.84030837004405286344 0.61828254847645429363 0.61831325301204819277 0.67706919945725915875 0.55360867751355861494 0.58684102983244789538 0.60331262939958592133 0.53106450336367233874 0.51618770461986176792 0.59395017793594306050 0.63712925668253387038 0.55225988700564971751 0.63909774436090225564 0.61242603550295857988 0.65703517587939698492 0.62153846153846153846 0.58806233460746839165 0.63578007256488975719 0.64532782653588022716 0.61430274135876042908 0.62412784154850326356 0.61790798432019806066 0.63968565815324165029 0.66754716981132075472 0.71353343865192206424 0.82727127062970677776 0.76001761338617349185 0.78284293410692084542 0.70906785034876347495 0.72899449035812672176 0.75122699386503067485 0.74359675508157870750 0.65455422461612669829 0.68164442841687702849 0.66530228812990673019 0.64529400594974365466 0.66092150170648464164 0.63646538400042637105 0.62311607944904681106 0.60365380223795387075 0.57512392218936143625 0.57243090281815715006 0.60734590065828845003 0.63138889922942337044 0.62741400384145276759 0.62715357070468211607 0.64217869775972749902 0.62127899333270729646 0.64937858508604206501 0.62577255871446229913 0.63539997863932500267 0.63727735368956743003 0.64259350393700787402 0.65117434187185127368 0.65983674829252040646 0.70927873956318694278 0.70574773934102068475 0.69597209816681553315 0.65163397414585492735 0.66749490079732987206 0.70033744131455399061 0.69210785800970873786 +35 0106000020E610000001000000010300000001000000C7000000FFFFFF9F2F5C5EC0000000604AFF4440FFFFFF1F26925EC0FFFFFF7F1D004540FFFFFFDF23CE5EC0000000E04C00454000000020C5E05EC000000040BEFF4440FFFFFF7F59F45EC0000000601EFF4440000000C0220D5FC000000060B8FF4440FFFFFF5F77165FC0000000E0A60C4540000000607C1A5FC0000000E07E1F454000000040F11B5FC0FFFFFFDF02374540000000C0FE185FC0FFFFFF1FCF4645400000006097195FC0FFFFFF1FBA4F4540FFFFFFBFBC235FC0FFFFFF7F936A4540000000E0FB1E5FC0000000E0517A4540000000C0AC185FC00000004081A1454001000000E9195FC00000006072A645400000000075115FC000000060C9BA4540000000C0620E5FC00000002076CD4540FFFFFFDF0D0A5FC000000040BBED4540000000607E075FC0000000A084224640FFFFFF3F67035FC000000040C654464000000000C2045FC0FFFFFFBF4E6846400000000068005FC0000000A0A4844640000000E024FD5EC0000000A085A54640FFFFFF1FADFE5EC0000000001DBE464000000020DEFB5EC0000000C00AC146400000000004F95EC0FFFFFF5FB3BC4640FFFFFFDFEDF65EC0000000A0E7BF464000000060F0FC5EC000000040CBC8464000000040D4FB5EC000000020FCD94640000000A06CFE5EC0FFFFFF9F48E34640000000201FFD5EC0000000E083EF4640FFFFFF3FB2FF5EC00000008096F8464000000040E0FA5EC00000006099014740FFFFFF1F78FE5EC000000000F819474000000000BEF25EC0FFFFFF7F490E4740FFFFFF3FA7F15EC000000040821247400000006076F45EC000000020CF184740FFFFFF7FA6F05EC0000000E0E41A474000000080D1ED5EC0000000A0C4154740FFFFFFDFD0EA5EC0000000805B164740000000A002E15EC0000000C03D1E4740FFFFFF3F30D75EC00000002079124740010000406CD35EC0FFFFFF3F8C124740FFFFFF1FD8CF5EC0000000C074124740FFFFFF5F84CD5EC000000040C8154740FFFFFF9F32CB5EC0FFFFFF3F85174740FFFFFF3F82C75EC000000020F9164740000000E028C35EC0000000A0F4134740FFFFFFBF44BE5EC0000000C0290E4740000000A081B95EC0FFFFFFDF2C0A4740010000E0F2B75EC0FFFFFF1F80034740000000209EB35EC0000000C0D6F846400000004085B35EC000000000BEF34640000000601AB25EC0000000401CEF4640000000A021B25EC0000000E0E0EC4640010000E05AB25EC00000000077E64640FFFFFF3FD6B05EC0FFFFFF9F5FE14640000000A05DB15EC0000000202ADD4640000000E098B05EC0000000C024D34640010000C07CAC5EC000000060CBD04640FFFFFF9F99A95EC0000000E0B1CD4640000000401CA45EC00000004028CC4640FFFFFF9FE69B5EC0000000E04FC8464000000080BC965EC0000000807DC84640FFFFFF1F53935EC00000004089C54640FFFFFF1F998F5EC0000000C02DC64640000000802C855EC0000000E09ACB4640FFFFFF9FEC7F5EC0000000201ACF4640FFFFFF7F2C7E5EC00000006066D14640FFFFFF7F3D7B5EC00000004033D24640FFFFFF1FC6785EC000000080A8D64640FFFFFF9FD4735EC000000040B5D94640FFFFFFFF7A705EC0FFFFFFDF4DD8464000000080226D5EC0000000A02FD84640000000A0C8615EC00000000020DC4640000000603F5C5EC0000000E0F9D8464000000020EF5A5EC0000000E06AD84640000000E076575EC00000008094D94640FFFFFF1F67545EC0000000C030D94640000000009D515EC000000000D9D64640FFFFFF3FA34D5EC00000000039D54640FFFFFF9FEF4C5EC00000002027D44640FFFFFF3F374C5EC0000000E083CE4640000000A0144B5EC0000000E0E2CC464000000000F0475EC000000040B9CD464000000060A1445EC0FFFFFF3FC9D24640FFFFFF3F11425EC00000008095D3464000000040E83D5EC00000008099D2464000000020A23C5EC0000000A042D3464000000060083A5EC0FFFFFF5F5CD14640FFFFFF3F0E375EC0000000E029D5464000000060882C5EC000000000F7DA4640FFFFFF1F102A5EC000000040CBDD464000000080D8275EC0000000A033DF46400000002069245EC000000060DBDE464000000020DF1F5EC0000000600FD94640000000604D1C5EC0000000403FD846400000002014125EC0000000E0BDDB4640FFFFFFDF330D5EC0000000C026DC4640FFFFFF7FE7095EC0FFFFFFDF75E14640FFFFFFDF51045EC000000080E2E346400100002090FF5DC000000040D8E746400000000097F75DC0FFFFFFDF79EA46400000004046F55DC0000000A0BEEB46400000000059EB5DC0FFFFFFBF24ED4640FFFFFF1FBEE75DC0000000A024F3464000000060A4E55DC000000040ECF44640000000A0B5E05DC0000000C01DF34640FFFFFFBF03DC5DC0000000800BF546400000004036D85DC00000000079F54640000000004ED35DC00000004066F74640000000205ECB5DC00000006014F6464000000080E7C85DC00000006082F64640000000A0FDC15DC0000000A0B3FB464000000000C9BE5DC0FFFFFFDFE5FF464000000060747F5DC0000000403A004740FFFFFFFFD27E5DC0000000A0000047400100000084665DC0000000200D00474000000000C35E5DC0000000E0FEFF464000000000C33A5DC00000000066FF4640FFFFFFFF6B395DC0000000A085FD464001000060C0365DC000000020ADF34640FFFFFFFF93325DC00000006049EC46400000006074315DC000000000F2E84640000000E0F02C5DC0FFFFFF7FDEE84640FFFFFF9FD1295DC0000000A0EFE3464000000060D1235DC000000080AFDF4640FFFFFF3FBF215DC000000000FDDA4640FFFFFF5FE4205DC0000000E011D54640FFFFFF5F0B1E5DC0000000A09DCD4640FFFFFF9F901E5DC00000006078C84640000000006D235DC0000000A033BF4640000000A025245DC0FFFFFF9FE0BA464000000060F62A5DC0000000A0F2AA464000000080052C5DC0000000A098A14640FFFFFF1F142F5DC00000004097914640FFFFFF7FA8305DC0000000409F8D464000000020BC315DC0000000C0BF8C46400000008051325DC000000040B6894640000000C09A335DC0FFFFFF5F62864640FFFFFF1FA0365DC0000000402F8246400000000037365DC00000006005804640FFFFFF9FB6365DC0FFFFFF7F737D4640FFFFFFFF37365DC000000000667C4640FFFFFF5F25355DC000000040837C4640000000202E365DC0000000803C7A4640010000E066355DC000000040CB754640FFFFFFDF6D375DC000000080326F4640000000205E395DC000000060A16C4640000000E0263A5DC0000000A01E6A4640000000E0D43C5DC00000000059634640FFFFFFDF77425DC0FFFFFFDFE65F46400000004031445DC0000000604D594640000000A003455DC00000000040584640FFFFFF9F49485DC0000000404C494640000000C025495DC0000000809A474640FFFFFFBF39495DC00000006073444640010000A0ED4B5DC0000000A086414640FFFFFF1FFD4C5DC000000000323E4640FFFFFF1F4C4E5DC0000000A08E3C4640FFFFFF5FD64D5DC0000000C0C736464000000020194F5DC0000000C0EE31464000000080D64C5DC000000080762B4640FFFFFF3FDA4D5DC00000000080264640000000A09A4D5DC00000008075244640000000A0DC4A5DC00000000071204640000000001B495DC00000008018204640FFFFFFDF25475DC0000000C08C224640000000205F465DC000000060332246400000000025455DC0FFFFFF1F3A1F4640FFFFFFFF43435DC000000060A71D4640FFFFFFDFE0415DC000000000EE1F4640FFFFFF1F783F5DC000000080A31F464000000080683E5DC0FFFFFF7FD61C4640000000A0C63E5DC0000000A056194640000000205F3A5DC0000000A0B5164640FFFFFF3FAE395DC000000020BE124640FFFFFF7F893C5DC0FFFFFFFFEB0B4640000000C0983D5DC0000000A0920B4640FFFFFFDF733E5DC00000002079094640FFFFFFBFAF3B5DC000000020D5014640000000C0E23D5DC0000000C04DFB4540FFFFFFBF5B3D5DC000000060DFF64540FFFFFF9F893E5DC000000080C8F34540000000C0893E5DC0000000A0D1EF4540FFFFFF9F063F5DC0FFFFFFFF02EE4540FFFFFF7FF9405DC00000000032ED4540FFFFFFDF9B405DC0FFFFFF5F81EB454000000060B4415DC0000000A074EA4540000000E04F425DC0000000E06EE64540000000A075415DC0FFFFFF1F7DE04540FFFFFF9F9E415DC000000060EED64540FFFFFF5F25415DC00000000059FF4440FFFFFFFFCB8B5DC00000004095FF4440FFFFFF7FD5D35DC0000000809FFE44400000002071D65DC00000004096FE44400000008083FF5DC0000000E0A1FE4440000000A0BB375EC0000000E06FFE4440FFFFFF9F2F5C5EC0000000604AFF4440 Oregon 41 0.57986111111111111111 0.58647342995169082126 0.57321225879682179342 0.56065088757396449704 0.57188498402555910543 0.64558823529411764706 0.63434903047091412742 0.63133640552995391705 0.58587987355110642782 0.66792452830188679245 0.63515016685205784205 0.59298928919182083739 0.70274914089347079038 0.71758664955070603338 0.86691776522284996861 0.86866791744840525328 0.82542579075425790754 0.78486055776892430279 0.84971751412429378531 0.90638766519823788546 0.88864265927977839335 0.79855421686746987952 0.82767978290366350068 0.79766374634960367126 0.78299959133633020025 0.77308488612836438923 0.78274633953304313415 0.75300109130592942888 0.73131672597864768683 0.77114610032954961553 0.79237288135593220339 0.78024606971975393028 0.77218934911242603550 0.77167085427135678392 0.78184615384615384615 0.78947368421052631579 0.79598102149037119732 0.78291171915332989158 0.76305125148986889154 0.77605221697051541751 0.75861357540746853724 0.77406679764243614931 0.79471698113207547170 0.81183078813410566965 0.82278481012658227848 0.84045207691178629091 0.85384721646636275729 0.87672796448953709575 0.86730945821854912764 0.86666666666666666667 0.85817154315923799107 0.81968003858831095747 0.78341146772448611612 0.74669529624907736697 0.74890815874422431799 0.73185437997724687144 0.72200607578745403187 0.70675716139335263000 0.68102306462662708381 0.66905485899945849127 0.66759700740396170097 0.68271244763614602035 0.70007072925585377657 0.68301030207787672429 0.68927775150327680562 0.70159177256648761955 0.70955019250633862334 0.70653083173996175908 0.69797168221148443645 0.69310050197586243725 0.68760814249363867684 0.69758858267716535433 0.68275030156815440290 0.69934080578758239928 0.71163734138402754847 0.69357239663178475416 0.68282931393815660755 0.66907867717520974713 0.65416280363434081216 0.65835900821596244131 0.66766535194174757282 +4 0106000020E610000005000000010300000001000000F600000001000080816A5EC0000000E0AD154340FFFFFFBF00725EC0000000C08E084340FFFFFF5FB5795EC0FFFFFFBF57094340FFFFFF3FF17E5EC0000000E0DD11434000000060CB8E5EC0000000C01B0943400000002067915EC0000000806A144340010000A023945EC0000000C05E1A4340000000E09E955EC000000000CA1843400100008031925EC00000000067144340FFFFFF5F63915EC0000000007D0C4340000000A06E995EC000000060A914434000000040669B5EC0000000E0930E4340FFFFFFDF389F5EC000000000870E4340FFFFFF7FC3A15EC0000000E04B134340000000204D9E5EC0000000E0F20A4340FFFFFFDF57A05EC0000000C06502434000000060349C5EC000000000D4FD4240000000C04A9F5EC0000000A046F7424000000060429D5EC000000020CAEA4240000000E0EFA05EC0FFFFFF1F3DE94240FFFFFF3F94AA5EC00000006018F442400000004033AC5EC0000000E07DF24240FFFFFFBF8CB45EC0000000E0FD00434000000080E2BA5EC0000000E0ED034340000000C026BD5EC000000080D3FE4240000000A09DC05EC0000000004DFF4240FFFFFFDF0ABC5EC0000000C0A0134340FFFFFF1F96BF5EC0000000400E264340000000400DC35EC000000020A925434000000020B5C75EC0000000008337434000000020FFD25EC0000000E0114643400000000075E15EC0000000E0FD604340FFFFFF1F21EE5EC0000000E06176434000000020ABEB5EC0000000005D8543400000008001F45EC00000006088AC4340000000C039F05EC0000000C0A7C64340FFFFFFDF12F25EC000000020F7D74340FFFFFFFF90F55EC000000080CBE94340FFFFFF7F6A005FC0000000E0D5FF4340FFFFFF7FFA055FC0000000A0DD0C44400000008006165FC00000004054204440000000C06F155FC000000000F2294440000000E00D195FC0000000A0BA374440000000E0ED065FC0FFFFFFFF3A7D4440FFFFFF5F81095FC0000000A082904440000000C081045FC0000000C030A84440FFFFFF1FA2035FC000000040AABA44400000004027095FC0FFFFFFFF19DD4440FFFFFF5F7B0F5FC0FFFFFF1F72E3444000000020340D5FC0000000609BEC4440000000C0220D5FC000000060B8FF4440FFFFFF7F59F45EC0000000601EFF444000000020C5E05EC000000040BEFF4440FFFFFFDF23CE5EC0000000E04C004540FFFFFF1F26925EC0FFFFFF7F1D004540FFFFFF9F2F5C5EC0000000604AFF4440000000A0BB375EC0000000E06FFE44400000008083FF5DC0000000E0A1FE4440000000A0B2FF5DC0000000E0BD964440FFFFFF3FB0FF5DC00000000040DC4340000000C0ADFF5DC0000000A0C7B84340FFFFFF3FA2FF5DC0000000C0E3A7434000000000A6FF5DC0000000C053944340FFFFFFDF95FF5DC0000000C0988D4340FFFFFFDF9FFF5DC00000008023884340000000A0A1FF5DC0000000E0417F434000000060DAF85DC0000000600F764340FFFFFF9FC7E45DC000000000FC5943400000006057D45DC0000000207B434340000000A0B1C95DC0000000A0B8344340FFFFFF5FA79A5DC0000000E080F14240000000E09D755DC0000000008BBA4240000000A0354A5DC000000000D67A424000000080A2F85CC00000004029004240000000E005E85CC000000040DAE5414000000080B2A75CC0FFFFFF5FDC7F4140FFFFFF1F6AA85CC000000000B37F414001000080B1A75CC000000020C8784140000000A04CA85CC000000020B2754140FFFFFFFF17A85CC00000006011704140FFFFFF5F71A45CC0FFFFFF3F7A6A4140000000C0A3A25CC00000002046614140000000A095A15CC000000020DC5F414000000000CF9F5CC000000000545F414000000000C09D5CC0FFFFFFDFDC5A414000000080F99A5CC000000080314E4140000000A0BE9B5CC000000080AA4C4140FFFFFF3F2C9A5CC000000020B74A41400000000011985CC0000000C0AD4441400000004084985CC0000000C0103D4140000000C00B985CC0FFFFFF5FD63A4140000000003D955CC0000000E0383A41400000004055935CC000000060C63741400000004016925CC000000040BE344140000000A073905CC0000000A0E6334140000000609A8B5CC0000000A0BE2E4140000000C0C4895CC000000040102B4140000000C088885CC0000000A042284140FFFFFFFFF6875CC0000000C0E4224140FFFFFF5F8B895CC0000000E02B224140000000200A8F5CC0000000A0D5174140000000A036925CC000000060EA154140000000E09B945CC00000008015124140FFFFFF3F339A5CC000000040230D414000000060169B5CC000000040060A414000000080679B5CC000000060D1034140000000601DA15CC0000000A086FB40400000000097A15CC000000020E8F9404000000060D59F5CC00000000067F64040000000804AA15CC0FFFFFFBF73EE404000000020B3A05CC0000000E0C4EB4040000000204DA15CC0000000C0BAE94040000000C03DA05CC0000000E0C6E24040FFFFFF9F9BA05CC00000008020DF4040FFFFFF3FAC9F5CC000000020ABDA40400000000048A25CC00000002063D74040FFFFFFFF90A15CC0000000802ED5404000000040B0A15CC000000060A1CF40400000008086A25CC0000000404DCA404000000020D5A15CC0000000A0AFC740400000008085A55CC0000000A034C14040FFFFFF1F3AA65CC00000008038BE4040FFFFFFFFB2A75CC000000020FABB4040010000604EA85CC0000000203EB84040000000403CA95CC0000000A0A4B540400000006058AE5CC0000000A09CB44040010000E0FAAC5CC0FFFFFFFF1AAD40400100004004AF5CC00000006020A740400000006052AB5CC0000000404DA24040FFFFFF7FF6AB5CC0000000E09E9E4040FFFFFFFF78AB5CC000000060BE9C40400000006059AB5CC00000004067954040FFFFFFDF5AAD5CC0000000C0A88F4040FFFFFFDF79AD5CC000000040348C4040FFFFFF3F71AA5CC000000020FA844040010000603DA95CC000000000AE8540400000000086A85CC0FFFFFF9F4A844040FFFFFF1FFCA65CC0000000807383404000000040BBA35CC000000000B48440400000002045A15CC0000000A08A834040FFFFFF3FED9D5CC0000000E0267D404000000040719E5CC000000080CA774040FFFFFF5F7B9D5CC000000060356C4040000000C0A0A15CC0000000C0A9674040000000E031A25CC000000040DD644040FFFFFF3FE0A15CC000000020BC624040000000C0B6A25CC000000040B6624040000000C0B3A25CC0FFFFFFBF5E61404000000020E4A35CC0000000E05E61404000000080D6A35CC0000000A0DB5F40400000004092A45CC0FFFFFF1FD85F4040000000208EA45CC0000000E0625E40400100004093A65CC000000000305E4040FFFFFF1F9AA65CC000000060F55C4040FFFFFF3F5EAC5CC000000080E55E4040FFFFFFDF8FAD5CC000000060135E40400000002029AE5CC000000080435C4040FFFFFFBFCA065DC0000000C0484F4040000000A024485DC0000000A09244404000000080BB4C5DC000000040F45B4040000000E0A9475DC000000020294D404000000020EA475DC000000080E5564040FFFFFF7FAA4C5DC0000000C0935E404000000060D44F5DC0000000A00B5740400000008034525DC000000040F36C40400000008041505DC0FFFFFF1FAE714040000000E0F6545DC0000000C0438E404001000080315A5DC0FFFFFF7FF59D4040000000402C665DC0000000207FB24040FFFFFFBFC5865DC0000000A0AFDF4040000000C0B98F5DC0000000600FE34040FFFFFF9F4D925DC00000000019DA404000000020DE995DC00000000085DE4040FFFFFF1F659B5DC00000006041E34040FFFFFFFFC8985DC0000000C0F9E7404000000020519A5DC0000000A004F14040FFFFFFBF9EA25DC000000060C40441400100006061B25DC00000000056024140000000400FBC5DC0000000202105414000000000C9CD5DC000000020BB1241400000004003D15DC0000000E0791E414000000000DADE5DC0000000A0FB2F4140010000C0BDE65DC0000000E04D354140010000E094F75DC000000080D0334140FFFFFF5FAC005EC000000000183B414000000060E8085EC0000000A0673C414000000020221D5EC000000040A4384140FFFFFFBF89205EC0FFFFFFDFBC42414000000080FA285EC0000000E042494140000000A070265EC0000000001E5A4140000000E05C285EC00000004045614140000000A0DB265EC000000080856D4140FFFFFF5F8E2A5EC0000000C0B0734140FFFFFF5F2C295EC000000020807C4140000000A068275EC0000000609489414000000040CB285EC000000040ED914140000000A00F375EC0000000A0C99A4140FFFFFF3F7C385EC00000000035A14140000000C055365EC000000000AAAE4140000000E0F2375EC0000000E0C1B64140000000606B3F5EC00000002072BA41400100006050495EC0000000A08ED04140000000003B515EC0000000C0EFD44140FFFFFF9FFE545EC00000006089E64140FFFFFF9F725C5EC000000020A0F0414000000080146C5EC0000000A030174240000000C065785EC0FFFFFF1F4B274240000000E0117D5EC0000000A0994A4240FFFFFF3F437A5EC0000000E0FA514240000000C071775EC0FFFFFFDFCA4D424000000020AE735EC000000040FA52424000000040A9705EC000000020D6684240000000009A725EC000000000D96C4240000000607A785EC0000000A0277B424000000040DB835EC0000000804979424000000000088B5EC0000000201E804240000000E081915EC0000000C0AC8D4240000000A0779A5EC0000000809D9E4240FFFFFFBFD7985EC0000000C01DAD4240010000202F9C5EC0000000A061BD4240000000404BA05EC0FFFFFF7FF0C24240000000E0D09F5EC000000020A4D94240FFFFFFFFD09F5EC0000000C039E44240FFFFFF1F97995EC00000006083E74240000000E01A965EC000000060D6DC42400000004060975EC000000020ECD9424000000020F3965EC0000000A00FCE4240FFFFFF9FA5855EC0000000E0EEB94240000000605A7E5EC0000000E0FABA424000000080E2855EC000000000AABF4240000000C0B68C5EC0FFFFFF5F1DDE4240FFFFFFDFEC935EC000000000A7E34240000000409D935EC0000000C027F24240FFFFFFDFB4975EC0000000E067F44240010000003B985EC0000000609CFC424000000020D8925EC0FFFFFF5FE7014340FFFFFF9FF87F5EC0000000605307434001000040AA6C5EC0000000800403434000000020076A5EC000000000080B4340FFFFFF3FDA645EC0000000600F0C43400000000062645EC0000000E02808434000000060F8625EC00000008022084340000000E097645EC0000000A0930E4340FFFFFFBF65635EC0FFFFFFBF9711434000000020256A5EC0000000A05B0C434001000080816A5EC0000000E0AD1543400103000000010000000B000000000000A07AF75DC000000000A1094140FFFFFF7FAFEA5DC000000040BB024140000000A095E45DC0000000C023074140000000C06AE15DC0000000606D044140FFFFFF7F75E25DC0000000C0D4004140000000808AED5DC0000000408EFB4040000000002AF65DC0000000E0F4FB4040FFFFFF9FD6F85DC0000000E098004140FFFFFF9FDFF75DC00000006014044140000000804FFB5DC00000002093074140000000A07AF75DC000000000A109414001030000000100000006000000000000A0A60A5EC0000000E04AF6404000000080340F5EC0000000C064014140FFFFFFFFEE025EC0000000E0420541400000006098FD5DC00000002050F9404000000000ED065EC00000002089F24040000000A0A60A5EC0000000E04AF6404001030000000100000008000000000000E001A65DC0000000A08ABD4040000000A022975DC0000000209BB44040000000C0CB925DC0FFFFFF3FCEAA4040FFFFFF7F66935DC0000000005BA7404000000020169D5DC0000000A091A9404000000060BF9E5DC000000000B3B54040000000A08DA35DC0000000409CB74040000000E001A65DC0000000A08ABD4040010300000001000000070000000000004067965DC000000020DA68404000000020D49A5DC0000000802D67404000000060B0A05DC0000000602E724040FFFFFF7F4FA65DC0000000C0AF8240400000004084A45DC0000000A099844040000000609AA25DC000000080617E40400000004067965DC000000020DA684040 California 06 0.86024305555555555556 0.85700483091787439614 0.85017026106696935301 0.85798816568047337278 0.87220447284345047923 0.88676470588235294118 0.91412742382271468144 0.88824884792626728111 0.83772391991570073762 0.96981132075471698113 0.86874304783092324805 0.82181110029211295034 0.87027491408934707904 0.82541720154043645700 0.97237915881983678594 0.98999374609130706692 0.96289537712895377129 0.95105293113261240751 0.95649717514124293785 0.97081497797356828194 0.96620498614958448753 0.90457831325301204819 0.94075079149706015378 0.92073425114726741761 0.91908459337964854924 0.92215320910973084886 0.94143252869014641868 0.90760276464168788650 0.91779359430604982206 0.95056755767118271695 0.96751412429378531073 0.96479835953520164046 0.94674556213017751479 0.94346733668341708543 0.95446153846153846154 0.96265804175242575713 0.95367010884733463578 0.94553433144037170883 0.92443384982121573302 0.94688273688948908395 0.93666185269238704353 0.94597249508840864440 0.94981132075471698113 0.95681937862032648763 0.95289216471719275757 0.96183766329076765008 0.97955518718054979970 0.99112238427393785669 0.98370064279155188246 0.98343558282208588957 0.98860632576793364324 0.96703915105715893561 0.95239812477461233321 0.92424344091793598604 0.91720994999683524274 0.90597269624573378840 0.90118850930021851516 0.87972851581994210999 0.85065083352363553323 0.82113550214520764777 0.80493855874714114044 0.81870885697187312986 0.81986375311767114619 0.79350445259298061812 0.77450847915681372880 0.76879994759596488930 0.76676996275080602247 0.76371295411089866157 0.75174176873806045623 0.75509986115561251736 0.76106870229007633588 0.79416830708661417323 0.77461623974076964923 0.78294186240213226720 0.79723100146233312892 0.79235264058299451221 0.79671217727872059887 0.78557878473784053351 0.77773039124791396254 0.77716035798122065728 0.77559921116504854369 +12 0106000020E6100000010000000103000000010000000801000000000040DD9555C000000040AF164340000000E0519755C0FFFFFF9FBB18434000000060DA9855C0FFFFFF5FED184340FFFFFF9FC69855C0000000A07F15434000000020F69555C000000040E7134340000000E0779555C0000000606D12434000000020059655C0000000A02D11434000000060329955C000000000C60F4340000000A00F9A55C000000000D80D434000000080F69C55C0000000C09910434000000060C09D55C00000002086104340000000E05B9E55C0FFFFFF5F4A0E4340000000A0529C55C000000060580B4340000000C0519C55C000000020B809434000000040569D55C000000080900743400000006033A055C0000000609A0643400000002039A155C0000000A00306434000000060C8A155C00000008062024340000000C0F9A155C0000000A063FE42400000004015A155C00000006099F84240000000E074A155C000000080C2F6424000000080A1A255C000000040F2F54240000000403CA555C0000000407AF64240000000004BA655C0000000C0E3F54240000000E058A755C0FFFFFF3FD0ED4240FFFFFF3F51A955C0FFFFFFDF47EC4240FFFFFFBF9EAA55C00000002075EC424000000080ECAA55C0000000A027EE4240000000C042AA55C0000000A085F34240FFFFFF7FCBAA55C0000000C0E1F44240000000E01AAC55C0FFFFFFBFB5F4424000000020A6AE55C00000002081F24240000000E03EB055C0000000A0FBF242400000008061B355C00000008047FD424000000060E2B455C0000000A0E9FE42400000000040B755C0000000A051FE4240FFFFFFFF9AB955C00000000011FA4240000000009FBB55C00000000010F84240FFFFFF5F4CBF55C0000000C01CF74240000000A0D7C055C0000000005DF64240000000C055C255C0FFFFFFDF37F442400000006090C455C0000000A04EE74240FFFFFFBFCFC655C0000000C060E44240000000C070C855C0FFFFFF7F14E54240000000001ECA55C000000080D8E94240FFFFFFFF3FCB55C00000000057EB42400000004083CE55C000000060AEEC4240000000A074D155C00000008075EF424000000020E0D355C0000000E063F2424000000080CDD855C0000000A0ABF7424000000020F2DC55C000000080DEF74240000000804EE055C0000000E031F5424000000000ADE655C0000000804DFC4240000000C022E855C00000006032F64240000000A00FE655C00000004003F24240000000E0FDE555C0FFFFFFFFB3EE424000000080E2E655C0000000E000EC424000000020B5E955C00000004000EA4240FFFFFF1FD2EB55C0000000E00CEB42400000004080EB55C000000020D1F24240000000A066ED55C0000000C029F34240000000809AEE55C00000006080F24240000000A03DF055C000000080F4F2424000000040B6F455C00000004069F04240000000C0DBF655C0FFFFFF1F09F242400000006089F955C0000000C057F64240000000E0FFFA55C0FFFFFF1FBEF5424000000020CEFB55C000000080BBF34240000000E0F4FB55C0000000E005F04240000000C040FA55C00000004056EB4240000000A0E3FA55C0000000C0A3E742400000002022FC55C00000004056E64240000000805BFD55C0000000E059E3424000000060B70056C00000004091E64240FFFFFFDF460256C0000000A020E7424000000080810556C000000080A7E8424000000080B60556C00000006066EA4240FFFFFF5FB20256C000000040ECE94240FFFFFFFF300256C0000000E0FFEB4240000000E0D80456C00000006014EF4240000000407E0656C00000006099F24240FFFFFFBF670656C0FFFFFF5FFDF34240FFFFFF1FDF0256C000000040B0F24240000000A0B30156C0000000E0EFF34240000000C0F20156C0000000A073F5424000000040600556C0000000803AF64240000000600D0556C000000000D5F84240FFFFFFBF220456C00000002003F74240FFFFFF5FAC0256C0000000A09DF7424000000080B80256C0000000E066FA4240000000A0630156C0FFFFFF9FCEFC4240000000A0DE0156C0000000E00D01434000000080630156C0000000C04A04434000000080A70256C000000020E704434000000000C20256C000000080C605434000000000390256C000000040EC064340000000406BFE55C0000000206209434000000060C0FD55C000000040620C434000000000CA0056C000000000D20B4340000000E02F0156C000000000390D4340000000E04DFE55C000000080DD10434000000020D6FC55C00000006086114340FFFFFFBFA5FB55C0000000E029144340000000A0AAFB55C0000000A0E71543400000006096FE55C0FFFFFFFFB0194340000000C01AFF55C0000000600E1E4340FFFFFF9FB8FE55C0000000E0DB1E43400000004042FB55C0000000C0022743400000004079FA55C000000040B3264340000000C080FA55C0FFFFFF5FF9234340000000A0DCF855C0000000007C264340000000608AF855C0000000006428434000000040F0F755C0FFFFFF7F8C284340000000803BF755C0FFFFFFBF86244340000000C067F655C0000000E09E2443400000008068F555C0000000801F2D4340FFFFFF5F2DF255C0000000606630434000000040E6EF55C0000000E07F354340000000004BEF55C0FFFFFF7F05394340000000E08DF055C000000020823A4340000000E063F055C0FFFFFFFFA93B43400000002057EC55C0000000E0A23D4340000000A083EB55C0FFFFFF3F8340434000000080D3E955C0000000800E404340FFFFFF5FB0E955C0000000A0F74143400000008011EB55C00000000012464340FFFFFF5FC8E955C0FFFFFF9F7449434000000080FFE855C000000040ED4B434000000040ABE755C0000000E0B24C4340FFFFFFBF3BE855C0000000C0BB4F43400000002003E855C0FFFFFF9F47524340000000A0A9E555C0000000A009564340FFFFFF1FCFE255C0FFFFFFFFCD5743400000004088E055C0000000004A5E4340FFFFFF1F83E055C00000004086624340000000C037E155C0000000E06A6343400000004081E055C0000000E0D4654340000000803BE355C000000060CF6D4340000000A0C7E355C000000000566F4340FFFFFF3F82E255C000000080D273434000000080EEE155C000000020497743400000006024E255C0FFFFFF9F5A7B4340000000E010E355C0000000E0107D434000000000E1E555C0000000203E7F4340000000603BE555C0FFFFFF7F747F4340000000E075E555C0000000E0FD874340000000202BE755C000000060D48A43400000002060E855C000000080638B4340000000406DE855C0000000004E8D43400000008062EA55C000000020868E43400000008034EA55C0FFFFFF3FB9904340FFFFFF9FE6EA55C000000060C6924340000000803BE955C0000000A091954340FFFFFF3FE8E655C0000000C0189943400000008007E655C0000000405C99434000000080ABE555C000000000AF9A43400000008069E555C000000020D79F434000000060D7E655C0000000800BA143400000004069E755C00000008005A443400000006014E755C0000000C019A64340000000E003E855C0FFFFFFFF58A74340FFFFFF1F40E655C0000000604CAB4340000000E092E255C000000000DEAC4340000000E077E255C0000000001DBD4340000000E046E255C0000000E0FECD4340000000204AE255C00000002093F143400000000043E255C0000000E0451544400000008048E255C000000000DB3D44400000000061E255C0FFFFFF5F4F3F44400000004017E255C0FFFFFF9F695F4440000000A00CE255C00000006045814440000000E007E255C0000000A03D964440000000A013E255C00000002091A64440000000E016E255C0000000A01FBC4440FFFFFF3FE9E155C0FFFFFF9F9EDC4440000000C0ACDD55C0000000A0F6D54440FFFFFF3FD5DA55C00000000092D64440000000A048DC55C0000000E03BD44440000000E042D955C0FFFFFFFF2BD14440000000E0F6CE55C0FFFFFFBF25D04440FFFFFFBF50BC55C000000020B5DB4440FFFFFF5F6DB555C0000000C0FAE14440000000409CA155C0FFFFFFBFFBE1444000000000038F55C0FFFFFF7FE5E14440000000005F8455C0FFFFFFBFDDE14440000000A0267355C0000000E0B9E14440000000C0346A55C0000000209CE14440000000E0055355C000000060BBE14440000000E05C4C55C000000000A4E14440FFFFFFFFDD3455C0FFFFFF7F83E1444000000020773255C00000008065E1444000000040963255C0000000E045D9444000000080A63255C000000080E5C3444000000020AF3255C0000000C0C3B6444000000000A03255C00000008052A4444000000000AA3255C00000000065A04440000000A0983255C000000060807E4440FFFFFFBFA03255C0000000400578444000000020C23255C0000000C0495D444000000040C53255C0FFFFFF3F5A4B4440000000A0DA3255C0000000402F2D444000000040E83255C0000000E0E3284440000000406E3355C0000000E0C8014440000000A0983355C0FFFFFF1F64F5434000000060C23355C000000020DBDD4340000000C0E83355C00000002031C84340000000E0E93355C0000000A0ADC1434000000000F93355C0000000C0F8A74340000000C0F93355C0000000E0C7A6434000000020F03355C0000000A01F8D434000000080FC3455C0000000C0438D4340000000A0C03855C0000000805188434000000080F63855C0000000C07986434000000000163855C0FFFFFFFF33844340000000A0083655C000000020BD80434000000060683555C0000000A0C97D4340000000E02A3655C0000000602F7A4340000000400F3855C0000000C0B976434000000000053855C00000004066744340000000E0063755C00000002074734340000000E0683355C000000020D5724340000000607A3255C0000000803171434000000060663255C000000020EC6E4340FFFFFF3FC43455C0000000A0CD6A4340000000C0673455C0000000608C65434000000040713E55C000000000EA634340FFFFFF9F9B4155C000000040D261434000000060624455C0000000E00B60434000000040A94755C0000000E0665B4340FFFFFF7F4D4A55C000000080F958434000000020224D55C0000000800E594340000000405F5155C0000000C0455F4340FFFFFF7F715555C000000040545E434000000040C45A55C0FFFFFF7F825E434000000040975C55C0000000A0C55C4340FFFFFFBF095D55C000000020E9584340000000202A5B55C000000000DA4A434000000060B85A55C0FFFFFF5FDC474340000000A0AC5B55C0000000E0BA444340000000E0D95D55C00000008051424340000000A0766055C000000060553C434000000020366755C0000000602A39434000000040316955C0000000801A31434000000080DF6955C000000040392B4340000000809C6B55C0000000608326434000000040CE6F55C00000006097224340000000E0517255C00000002023244340000000209F7355C0000000409F244340000000A0C17555C0000000405B234340000000408D7655C0000000E0861E434000000000607A55C00000000008174340000000E08B7A55C0000000804B08434000000000947B55C00000006059044340000000005A7D55C0000000A081014340000000C06D8055C0FFFFFF7F37004340000000A0068255C0000000A016FF4240FFFFFF3F608355C000000040BDFB4240000000A0B88655C00000002071014340FFFFFF9F338C55C0000000A043024340000000C0239055C0000000203405434000000040C69155C0000000007007434000000060A79255C0FFFFFFBF090A4340FFFFFF7F0D9355C0000000003B13434000000040DD9555C000000040AF164340 Indiana 18 0.52690972222222222222 0.49661835748792270531 0.49716231555051078320 0.45857988165680473373 0.46964856230031948882 0.52794117647058823529 0.58310249307479224377 0.55414746543778801843 0.57639620653319283456 0.59371069182389937107 0.57619577308120133482 0.53651411879259980526 0.62199312714776632302 0.58600770218228498074 0.71437539234149403641 0.74921826141338336460 0.76094890510948905109 0.68525896414342629482 0.74180790960451977401 0.80176211453744493392 0.75401662049861495845 0.73445783132530120482 0.77385798281320669380 0.74176053400083437630 0.79403351042092357989 0.74741200828157349896 0.75464978235061337554 0.73044743543106584212 0.72669039145907473310 0.73782497253753203955 0.75247175141242937853 0.75495557074504442925 0.73833004602235371466 0.75345477386934673367 0.76492307692307692308 0.76947956483387239047 0.79709740440971253140 0.78497676819824470831 0.75160905840286054827 0.76547377897816790457 0.76624716319372807922 0.74852652259332023576 0.77452830188679245283 0.78199052132701421801 0.81717673449767665438 0.79847350653163070600 0.80535985633374775521 0.82637920101458465441 0.82633149678604224059 0.81860940695296523517 0.80339075745146294777 0.75962697966074443283 0.74684457266498377209 0.71784204522579346440 0.70909551237420089879 0.70790671217292377702 0.70047433779246389170 0.68973949495957680407 0.66974195021694450788 0.65047694422460115800 0.65007559018490522154 0.65922351885098743268 0.67211406023154524811 0.67291775798847564170 0.67941355313830146612 0.69281409668544477925 0.68378877515885685667 0.68041945506692160612 0.66715361276547926733 0.67238064722845241910 0.66521628498727735369 0.66464074803149606299 0.65256983372359800374 0.66773755979153280503 0.68609368366432378886 0.66933280010664533760 0.65433201480158223810 0.63178377955527741380 0.61470424624513257927 0.62312940140845070423 0.62905794902912621359 +18 0106000020E610000004000000010300000001000000F2000000FFFFFFFF81ED52C000000080245343400000006046ED52C0FFFFFF5F5B474340000000E0BEEC52C0000000C0453B4340000000C069D652C000000040443A4340FFFFFF3FF5C552C0FFFFFF5FA839434000000080EFC952C0000000E04F2F4340000000C0A9C952C0000000C00A234340FFFFFFBFD2D052C000000000C819434000000040E6D752C000000060D2084340000000A0DBD752C0FFFFFF9F23024340FFFFFF7F17E852C0000000A08AFF4240000000607CE952C0000000402DFC42400000002068F752C00000006065FD42400000006040F152C0000000A0720C43400000008075F952C00000004064164340FFFFFFFFA2F552C0000000E0A41D4340000000A025F752C0000000C0B91E4340000000C0D3F252C0000000C0BD2143400000000047F952C0000000C022214340FFFFFFFFD5F752C0000000C0B92D4340000000C0C1F852C0000000200F304340000000A0CBFC52C0000000601A244340000000C0B2FF52C0000000C029244340000000E0540153C0000000E03329434000000080300453C0000000C024214340FFFFFF5FD51253C000000080ED374340000000E0AF1253C000000000473D4340000000404D0C53C0000000408C45434000000040111053C0000000A02B4C4340000000400B0253C0000000C034494340000000C0CB0153C0000000E09D4F4340000000C0000353C000000000C24B434000000020DD0453C0000000202E4E434000000040F00753C0000000009F5A434000000080230B53C0000000A0C35A4340FFFFFFBF4A0E53C000000020A3614340FFFFFF7F181153C0000000A08B624340000000A09B1553C0000000C0F5564340000000206F1653C0FFFFFF9F79594340000000406F1153C0FFFFFF7FC06A4340000000C07D0C53C0000000E0F3614340000000609D0A53C000000020F1644340FFFFFFDF510753C0000000A05671434000000000DB0453C0000000C0DC714340000000E0950653C000000040F2724340000000A01C0653C0000000605C794340000000A0490753C000000000DA75434000000060C70C53C0FFFFFFDF967C4340000000401F0753C0000000202E8F434000000080310E53C0000000C0E48B4340000000004A0F53C0FFFFFFDFBE904340000000E0FA0D53C0000000A0389A434000000020310753C00000004020A9434000000020650253C000000000DFAD43400000002062F652C0000000A087B04340000000A0A4FE52C0000000A080B2434000000000F8FC52C0FFFFFF9F4FBC43400000008062FE52C00000002013C34340000000A0020253C000000060F3C8434000000040060553C0000000206CC54340000000C0E30953C0000000A072B34340000000A0810E53C000000060FCAF4340000000404C1753C00000000057B24340FFFFFFDF891953C000000020969D4340000000C0002253C0000000200E9F434000000060A82653C00000008032A14340000000202B2453C000000000A09D434000000040ED2453C0FFFFFFFF5B99434000000080DE2653C0000000802A974340000000C0162653C0000000E04F94434000000040172453C0000000401F994340FFFFFFBF211B53C000000040268F4340000000E0351E53C00000002041744340000000E0242353C00000000026614340000000C09D2153C000000040D55A4340000000C0912053C000000020D442434000000020B11853C0000000801532434000000000F31953C0000000604A2C434000000040F91A53C0FFFFFF1F0629434000000000341E53C0000000E0F62A434000000000492153C0000000607F344340000000A06C2953C000000080A739434000000080001653C0000000A0451B434000000080211553C0000000A0D905434000000000F22453C0000000607F1C4340000000C0A73053C0000000E0FC1D4340000000C04E3753C0000000A01732434000000020263A53C0000000806126434000000020463E53C0000000205F2A4340FFFFFF3F274053C000000020A3364340000000A0234E53C0000000400132434000000040605053C0000000A0F0344340000000C0C65153C000000040593E4340000000C0514853C0000000C0F5524340000000E0014853C000000020C256434000000020F64553C0FFFFFF1F1C5A4340FFFFFF7F3D4553C0000000408E5B4340000000E0A74353C0FFFFFF7F235B434000000060F94253C000000000015C4340000000A0E84253C000000020E164434000000040513A53C000000020EB714340000000E0864053C000000020B87B434000000080B64253C0000000C0287F434000000020D94753C0000000C04D774340FFFFFF1FBB4953C0FFFFFFDF7D7B4340FFFFFF3F994F53C0000000A0E97C434000000020625053C0000000808783434000000040C65453C0000000E002884340000000602D5653C0FFFFFFFFC4884340000000E0B65B53C0000000208C884340FFFFFF9F6B5D53C0000000E0588A434000000000AC5E53C0000000804E8D4340000000C0D56053C000000080EE8E434000000040106153C0000000202794434000000000A25E53C0000000C0A5964340FFFFFF5F915D53C000000020FC9B4340000000E0C15D53C0000000C0519D4340000000C09E5F53C000000020FD9F4340FFFFFF3FB36253C0000000A06CA24340000000E0696453C0000000C031A6434000000020756753C0000000205DA64340000000607E6B53C000000080CAA84340FFFFFF7F936E53C000000040AAA8434000000020067053C0000000E0D1A94340000000204B7053C0000000C053AB434000000020BA6F53C0FFFFFF5F1DAE4340000000A06A7053C0000000A06EB0434000000080336F53C000000040B3B24340000000806E6F53C0000000A0A0B3434000000020777053C00000008068B64340000000205D7353C0000000E052B74340FFFFFFBF847353C0000000404FB8434000000000F37253C000000060B4B94340000000807C7353C0000000E044BB4340000000E0437253C0000000A0C0BA4340000000A02F7353C00000008088BD4340000000C0657153C000000000BFBF434000000080D47453C00000002035BF4340FFFFFF3F487653C0000000803EC04340FFFFFFBFD57453C00000004087C14340000000C0117553C000000060BBC34340FFFFFFDF7C7553C00000000044C3434000000040067653C00000002013C44340000000C05A7753C0000000E0DCC1434000000080AB7753C000000040DDC5434000000000F97853C0000000606CC7434000000040AB7853C0000000C03CC84340000000809B7653C0000000A05DC8434000000000C57553C0FFFFFF5F4CC94340000000A0EE7553C0000000C079CD434000000080C57653C0000000A010CD4340000000C0E07853C0000000A0E8CE4340FFFFFF7F067953C0000000A0E0CC434000000080D37953C0000000A04ACC4340000000C0167C53C0000000A01ECF434000000020A97C53C0000000A0B5CE434000000020E37B53C000000080C1CB4340000000207F7C53C000000040FFCA4340000000A0BA7D53C0000000C03CCE434000000000B67F53C0000000E0A7CC434000000060B58153C000000000B7CF4340000000200E8653C00000000077D64340FFFFFF3FBA8B53C000000000E7D84340000000C0178D53C00000002082D6434000000060958E53C00000004042D6434000000020B08E53C0FFFFFFDF48D4434000000000839053C0000000C00ED24340000000807D9153C00000000025CF4340000000C0489653C0000000E0FBD14340000000E0D99653C000000080EED0434000000020309853C000000000CCD0434000000040A29853C0000000C0A4CE434000000080A89B53C00000006078CF4340FFFFFF7FDF9953C0000000E033CB434000000080DB9C53C0000000C0DACB4340FFFFFFBF929D53C0000000C055CA434000000020F39A53C00000000050C64340FFFFFF7F8D9C53C0000000402CC64340FFFFFF5F309D53C0000000604EC44340000000A0D19E53C0000000408AC243400000004094A053C00000006035C343400000000020A453C0FFFFFF7FAFC24340FFFFFF3FB2A653C0FFFFFF3F8EC4434000000040CAA853C000000060D2C34340FFFFFFDF8FA953C000000020DAC4434000000000ABAA53C0FFFFFF1FB7C4434000000060DCAD53C0000000209DC74340FFFFFF1FE9AE53C000000080CCC94340000000E0BFB053C00000004075CA43400000004089B153C0000000E0FECC43400000000023AF53C000000000EACD4340000000C0C4AE53C0000000E08BCF4340000000A0E3AE53C0FFFFFF7F3DD04340000000E024B153C00000000032D043400000004078B153C0FFFFFF9F73D243400000002019B353C000000040BCD04340000000601EB353C000000020C3CE4340000000A0A6B453C000000040F5CA4340000000C0A2B353C0000000C08AC84340000000E0A7B553C0FFFFFFDF17C84340000000A0BFB753C0000000204AC343400000004029BD53C000000040EDBA4340000000C01FBE53C0FFFFFF7F1EB843400000008024C353C0000000A0EABD43400000006023C453C0000000802CBE43400000004089C453C0000000C041BC4340000000C0B5C653C00000008042BC4340000000A034C653C0000000E075BB434000000040ADC653C0000000403EB94340FFFFFFDF6CC853C0000000205EB54340000000C022CA53C000000080F9B44340000000E072CA53C0000000E05AB2434000000080AAD053C0000000009DAC434000000080ECD153C0000000409EA94340000000A0E9D253C0000000C074A64340000000E02BD653C0000000E05FA5434000000020A5D853C00000000075A24340000000C0C4DC53C0FFFFFF5F229B4340000000E087DD53C0000000C0489B4340000000A05DDF53C0000000C040994340FFFFFF1FCCDE53C0000000602FDC43400000000066D953C0000000E00FDC4340000000208CBB53C0000000C072DC4340000000605AB453C0000000408CDC4340000000A0A49853C0FFFFFFDFA0DC4340000000A06D9553C000000040ACDC434000000080288653C0000000E0D8DC434000000020785E53C0FFFFFF7F19DC434000000000BE5D53C00000004028DC4340000000802A4E53C0000000003CDC4340000000A0D03F53C0000000E042DC4340000000609C3253C0000000C04EDC4340000000407D2453C0000000402EDC4340000000C0F00E53C00000004062DC434000000060EE0853C0000000806EDC4340000000E0A6F252C000000000A4DC4340000000E073F152C00000006006B1434000000040F0F052C0FFFFFF5FDEA5434000000000BFF052C000000060B69F4340000000E02FF052C0000000401E924340FFFFFF5F65EE52C0FFFFFFFF426A4340FFFFFFFF81ED52C000000080245343400103000000010000000E000000000000A0C21253C0000000403074434000000040D91253C000000040D97B434000000040B31553C0000000E0737A434000000000201453C0000000009178434000000000A61453C000000040BF74434000000040EC1553C00000008047764340FFFFFFFF191553C0000000201D704340FFFFFF3F091853C000000040536D4340000000C0D51653C0000000E0A57A434000000020301353C00000004032854340000000A0E10F53C0000000804C7D434000000080CA0F53C0000000E03776434000000040811153C00000006080794340000000A0C21253C0000000403074434001030000000100000004000000000000005FC452C00000006098394340000000A0F1C252C0000000E088394340FFFFFFBF9CC552C00000004056294340000000005FC452C00000006098394340010300000001000000080000000000008053D152C0000000008803434000000040A5CF52C000000000DA044340000000A06CCD52C0000000000E0C4340000000208BCA52C0FFFFFF5F381A4340000000A00AC652C000000060FC284340FFFFFF5F15CB52C000000060E50F43400000008086CF52C0000000C0A60343400000008053D152C00000000088034340 Maryland 24 0.66666666666666666667 0.68792270531400966184 0.72417707150964812713 0.75739644970414201183 0.74440894568690095847 0.76911764705882352941 0.75900277008310249307 0.71198156682027649770 0.70073761854583772392 0.79622641509433962264 0.73748609566184649611 0.69133398247322297955 0.74742268041237113402 0.71694480102695763800 0.80790960451977401130 0.82801751094434021263 0.79805352798053527981 0.74843483210017074559 0.76553672316384180791 0.82599118942731277533 0.82880886426592797784 0.79132530120481927711 0.82089552238805970149 0.81101376720901126408 0.82427462198610543523 0.80248447204968944099 0.81005144440047487139 0.79447071662422699163 0.80676156583629893238 0.82680336872940314903 0.82062146892655367232 0.82262474367737525632 0.82412886259040105194 0.82851758793969849246 0.83753846153846153846 0.84710379300205821817 0.85263745464694390176 0.84770263293753226639 0.83814064362336114422 0.86225523295070898042 0.86837218898287600578 0.89842829076620825147 0.92339622641509433962 0.92873442162541688608 0.93286332318538695722 0.93673858799354175840 0.95013123359580052493 0.95573874445149017121 0.94019742883379247016 0.92658486707566462168 0.91468416735028712059 0.90280569177586622719 0.89455463397042913812 0.88881433268469435684 0.90056332679283498956 0.89266211604095563140 0.90395992112135585994 0.89659646671324483481 0.87764329755651975337 0.85916607656100304078 0.85285110671783540722 0.86112357869539198085 0.87745225775229870082 0.85063733193644141785 0.84805080737787987298 0.85307218655836499410 0.84189438757942842833 0.83185946462715105163 0.82093493650972019328 0.82372102958453487130 0.82608142493638676845 0.83346456692913385827 0.83800468317604484496 0.86368720401703909950 0.87997075333742157649 0.88095714190495234286 0.88373527285100591213 0.86030547787143332095 0.84974967550528462822 0.86267605633802816901 0.89424681432038834951 +33 0106000020E610000001000000010300000001000000D5000000FFFFFF5F76D154C0FFFFFF9FF94D4340000000A091D254C0FFFFFF1F5C4C4340000000C09FD354C0000000A0514C4340000000C081D454C000000060A14D4340000000A020D554C000000080E2504340000000E0C6D754C000000080D45343400000008009DD54C000000020F4544340000000A002E054C00000000054584340FFFFFF7FB4E154C0000000C017594340000000E094E754C000000060C5564340FFFFFF5F88E854C0FFFFFF5F1B554340000000602BE954C0000000806151434000000040F9E954C0000000E0D84F4340000000606EEB54C0FFFFFF1F784F434000000040A0ED54C00000006057514340FFFFFF9F4CF154C0000000A04B5343400000004098F254C000000080CD584340000000609BF554C0000000801C5B434000000060E3F654C000000020575F4340FFFFFF3F68FA54C000000080026143400000004095FD54C0000000A087634340FFFFFF9F720355C0000000C0BF61434000000020B10555C0000000C0F9614340FFFFFFFF500B55C000000040EB64434000000020A40E55C0000000000468434000000020100F55C000000040EF6F4340000000E0BD1055C0000000C06D754340000000A0921255C000000080E4784340000000600E1455C000000020CB81434000000040221655C000000000D5844340000000400C1955C0000000409184434000000000DE1A55C0FFFFFF1F0D864340000000603F1B55C000000040D68A4340000000807A1C55C0000000604E8E4340000000C07E1F55C000000020BC8D4340000000A0FB2055C0000000A00C8C4340000000C0F52555C000000080FC88434000000060DA2755C0000000809589434000000000B92A55C0000000E0768B4340FFFFFF1F8C2F55C0000000402D924340000000008F3255C000000060B18D434000000020F03355C0000000A01F8D4340000000C0F93355C0000000E0C7A6434000000000F93355C0000000C0F8A74340000000E0E93355C0000000A0ADC14340000000C0E83355C00000002031C8434000000060C23355C000000020DBDD4340000000A0983355C0FFFFFF1F64F54340000000406E3355C0000000E0C801444000000040E83255C0000000E0E3284440000000A0DA3255C0000000402F2D444000000040C53255C0FFFFFF3F5A4B444000000020C23255C0000000C0495D4440FFFFFFBFA03255C00000004005784440000000A0983255C000000060807E444000000000AA3255C00000000065A0444000000000A03255C00000008052A4444000000020AF3255C0000000C0C3B6444000000080A63255C000000080E5C3444000000040963255C0000000E045D94440000000C09A1855C00000004082DA4440FFFFFF1FFE1655C0000000609FDA4440FFFFFFBF98F754C000000000A4DB4440000000A0E5F054C000000060C6DB444000000080E5DE54C000000080CFDC444000000040D8C954C00000002022D04440000000A039C054C0000000E0E1C4444000000060F0B254C000000000D0C44440000000203AB254C000000060F1C04440000000C082C454C00000002060BA4440000000602DBA54C000000020F7B6444000000000E4AD54C000000020A9B94440000000E021A354C0000000C015B2444000000020DB9554C0000000003AB74440000000E0018154C000000060F4C1444000000020927D54C0000000803DC0444000000000466F54C000000080DCBE4440000000809E5E54C000000080DAD0444000000020325754C0000000E0B3DC444000000040FF3F54C000000080D3EC4440FFFFFFBF542154C00000000050FE444000000060762154C000000060E4EC4440FFFFFF5F8B2154C0FFFFFFBF5DBF4440000000004B2154C0000000409FBE4440000000207B2154C0000000C095904440000000405A2154C0000000C0D7724440FFFFFF1F502154C000000060536D4440000000E06B2154C0FFFFFFBF8D514440000000A0C62454C000000020D64E444000000000272754C0FFFFFF1F5C4F444000000080CD2854C0000000E0944E4440FFFFFF5FBF2A54C000000060814A444000000000CE2A54C000000040BB484440000000A08D2854C0FFFFFF7F0245444000000080072854C00000002090404440000000C0872654C0FFFFFF1F803D444000000000322854C0000000E0F6324440000000E0482854C0FFFFFF7FBD31444000000040012754C000000040C52F4440000000C0B32654C0FFFFFFBF30274440FFFFFF5F5A2754C00000002062234440000000C09E2954C000000020701F4440000000C0702B54C0000000A0D7184440FFFFFF9FDE2C54C0000000A084154440000000E0F12C54C0FFFFFFDFB613444000000080422F54C0000000408E044440000000204D2F54C000000020E0FD434000000020D93054C00000006035F94340000000C0943054C0000000A0E9F54340000000202C3154C000000000E5F4434000000020F53254C0000000E0BAF54340000000C0B63354C0000000E039F5434000000020FD3354C000000060D1F3434000000060A03254C0000000A0A6EF4340000000201E3354C0000000A0A6ED4340000000E0DE3454C0FFFFFFBF77EB4340000000406F3454C0000000E08AE7434000000000BD3754C00000000045E1434000000020D33654C0000000C03DDE434000000060473554C00000004000DC4340000000604F3554C00000008006DA434000000020453754C00000004013D7434000000000DE3754C000000060C7D4434000000000673854C0FFFFFF5FDFCF4340000000C06A3A54C000000040BBCD4340000000C0B23B54C0FFFFFFBFADCD4340FFFFFFDFF63E54C00000002076CA434000000060184254C000000000A4C5434000000040674254C0FFFFFFDF2BC44340000000604C4654C0FFFFFF3F89BF434000000020814754C0000000E0DDBB434000000020914B54C00000006007B8434000000080D44C54C0000000A039B5434000000040684E54C0000000A042B44340000000E0374F54C000000000B7B1434000000000305254C00000002089B14340FFFFFF1FB25554C0FFFFFFBF41AD4340000000A0115854C0000000203DAC4340000000E0C85B54C000000020F6B34340000000E0AD5C54C0FFFFFF1F9AB4434000000040C55D54C00000008011B44340000000809C6254C00000002023AD4340000000C0AE6354C00000000092AA434000000060A96454C00000002007A2434000000020BB6A54C0000000209DA24340000000A0236C54C0000000A04CA14340FFFFFFDFAC6C54C000000020279C434000000040496E54C0FFFFFFDF499B434000000060947054C0000000807C964340FFFFFF9FAB6F54C0000000201A904340000000C03C7054C0000000401D8C4340FFFFFF1F567254C000000000E189434000000020767454C000000020D989434000000040C37454C0000000C07D88434000000020127454C0000000C0A285434000000020A77154C0000000E024824340000000400A7254C0FFFFFF1FF67B4340000000E0CB7054C0000000800D774340000000C0227254C0000000A03476434000000020BB7454C0000000C064794340000000E0D37554C0000000200A784340000000007C7754C0000000405C71434000000040247954C0000000A0CA6F4340000000C0957A54C0000000E032714340000000C0A57B54C0000000408472434000000020857954C00000008050774340000000E0637B54C000000000FA7D434000000020067C54C000000040DC7E4340000000C06B7E54C000000040187F434000000000FD7F54C0FFFFFF7FF1814340000000E0C08254C0000000C0CC814340000000C0C08354C000000020977E434000000020738554C0FFFFFF3F127D4340FFFFFF5F7C8654C0000000A0DB794340FFFFFFBFEC8854C0000000E01C734340000000E05B8954C0000000C05A6B4340000000A0A98C54C000000020FB66434000000060E18D54C0000000A0B163434000000060C88B54C000000080E85A4340FFFFFF5F1A8C54C000000080C2564340FFFFFF5F1F8B54C0FFFFFFDFE8504340000000E0CC8B54C000000040274C4340FFFFFFBFAE8D54C000000020D94A434000000080589154C0FFFFFF9F224C434000000000919254C0000000603D4A4340000000A01E9454C0000000E0893B434000000060139554C0000000208F384340000000E0459954C000000060D5364340FFFFFF9F8F9A54C00000006014374340000000E0AF9F54C0000000A0EF334340FFFFFFFF0CA354C00000004041334340000000A0D5A454C000000060B0334340000000E08CA554C0000000C0CA3443400000008049A754C0000000C07D3C434000000040DFAA54C000000080434043400000004086AC54C00000000000454340000000E07DAF54C000000040C8464340000000C05BB354C0000000A05247434000000040EFB454C0000000A02949434000000060A7B654C0FFFFFF3FD94C4340000000800CB754C0000000207F534340000000E053B854C0000000E07357434000000020E4B754C0000000E0055C4340000000A0FCB854C0000000C0105F434000000060F8BA54C000000000885F4340000000E03EBE54C0000000C01A5C434000000020BBC154C0000000A0725B434000000020E7C354C000000060C35743400000004020C754C0000000C016554340000000002BC954C000000000444F434000000080A6CB54C0000000C00C4E4340000000E0AFCF54C000000060E24F4340FFFFFF5F76D154C0FFFFFF9FF94D4340 Ohio 39 0.66927083333333333333 0.63864734299516908213 0.63904653802497162316 0.59171597633136094675 0.61501597444089456869 0.66911764705882352941 0.71468144044321329640 0.68317972350230414747 0.68282402528977871444 0.70566037735849056604 0.68409343715239154616 0.64070107108081791626 0.70532646048109965636 0.65532734274711168164 0.78656622724419334589 0.82051282051282051282 0.81508515815085158151 0.74558907228229937393 0.79152542372881355932 0.84746696035242290749 0.80664819944598337950 0.77493975903614457831 0.82994120307553143374 0.79933249895702962036 0.82713526767470371884 0.81366459627329192547 0.82588049070043529877 0.79374317933794106948 0.79822064056939501779 0.79714390333211277920 0.81709039548022598870 0.81715652768284347232 0.79059829059829059829 0.78988693467336683417 0.80123076923076923077 0.80976183475448397530 0.82277421155456321518 0.82111512648425400103 0.78879618593563766389 0.81386450596443844249 0.81163606354446049103 0.80569744597249508841 0.81660377358490566038 0.82341583289450588029 0.83608396090370132992 0.84147952443857331572 0.84086199751346871115 0.85643627140139505390 0.86214416896235078053 0.85132924335378323108 0.84322304256676693100 0.81220355334030066726 0.79206635412910205554 0.77065020465678051399 0.77011203240711437433 0.76501706484641638225 0.76187176890689122209 0.74523405529493961473 0.71591687599908654944 0.69725496730120381555 0.69097181842849943792 0.70287253141831238779 0.71537058407474965566 0.70689715383272219312 0.70937774474697655564 0.72261889165465740862 0.71640529627195041788 0.70545530592734225621 0.69988200921451848522 0.69860087578767489053 0.69089058524173027990 0.69881889763779527559 0.68511554199484377587 0.70255348516218081435 0.71571772253408179631 0.69408340554111399942 0.68259538088554293735 0.65795083409930964348 0.64554051548303356202 0.65142678990610328638 0.66402457524271844660 +3 0106000020E61000000100000001030000000100000051010000000000E0889D57C0FFFFFFFF2B194140FFFFFF3FF49C57C0FFFFFFDF10414140FFFFFFFF869C57C000000000255E4140000000401A9C57C000000020EB764140000000E0699B57C00000002042B34140FFFFFF1FF89D57C0000000600CD24140FFFFFFFF159F57C0000000204FE14140FFFFFF5FB3A257C0FFFFFF1FAA0D4240FFFFFF9F62A357C0000000800C154240000000E0DCA657C000000080463D4240FFFFFF7F7DA757C0000000A0A23E4240000000A02C8557C000000080D73E424000000060DE7657C000000000AF3E424000000020296657C0000000C0B43E4240000000A0005557C0FFFFFFBFBE3E424000000060045357C0FFFFFF7FCC3E4240000000E0883657C000000060B23E424000000000C23157C0000000A0B53E424000000000772157C000000060D43E4240FFFFFFBF5A0957C0000000A0EC3E4240FFFFFFBF280857C000000040E53E4240000000000FEC56C0000000A0D73E424000000080FBDC56C0000000A0C43E4240000000A058DA56C000000060DA3E4240000000A090C856C000000040753E4240000000E079B356C0000000409E3E42400000004039A556C0000000E0D73E4240000000205C8E56C000000080123F4240000000409C8956C0000000C0F33E424000000020C98856C0000000A08C3A4240FFFFFF9F808756C000000040193A4240FFFFFFDFEC8756C0000000A016364240000000207A8756C000000040D4334240000000A0218556C0000000A0DD32424000000000558356C000000080F7304240FFFFFF7F368356C0000000E0692E424000000020548456C000000080A4294240FFFFFF1F2F8356C0000000E075264240000000E03A8456C0000000E0D9224240000000E0088756C00000000006214240000000E0658856C000000020251B4240000000E0528A56C00000006035194240000000C0078E56C0000000A016164240000000C0DC8E56C000000080A0144240000000A0078F56C0000000208C11424000000080E09056C0000000A0330F424000000060399256C000000000D60E4240000000E02C9456C000000060BB0B4240FFFFFFDF409856C0000000C0AAFE414000000020249256C000000040DEFE414000000020A57D56C00000006098FF414000000040316E56C000000000FCFF414000000000A46D56C0FFFFFF7FB0FB414000000020826A56C0000000800AF84140000000404E6956C000000060F7F44140000000C08E6956C00000000078F24140000000A0896A56C0FFFFFFDF5CF1414000000060BD6D56C0000000A0A9F44140000000003B6F56C0000000201FF54140FFFFFF7FD37056C0000000C0CAF24140000000A00A7156C0000000402AF14140000000607E7056C0000000A08AEF414000000060E46C56C0000000E0C7EB414000000060DA6C56C000000000ECE94140000000A0196F56C0000000004DE7414000000080A07056C000000040A1E84140000000A0957256C0000000601CE74140000000A0317356C0000000C019E3414000000040EE7456C0000000000FE1414000000020087756C0000000C0C4DF4140FFFFFFDF397A56C0000000809EE04140000000C0DD7C56C000000080FCDD414000000020EE7C56C0000000C032DB4140000000E0807B56C0FFFFFFDF8FD64140000000802D7956C0000000E0F6D34140000000205F7756C0000000E02ED6414000000020DD7656C000000060E5D5414000000040597656C0000000A096D2414000000020497756C0FFFFFF7F9BD04140FFFFFFFF277856C00000002011D1414000000040407D56C00000008032CD414000000060507D56C0000000600FCA4140FFFFFF7FFC7A56C000000040E5C5414000000060987B56C0000000E0BDC34140000000A0A47C56C00000008071C34140000000E0957D56C00000002022C4414000000060557F56C000000000E5C74140000000801D8256C0FFFFFF7FB5C64140000000209E8256C0000000007CC5414000000020AD8256C0000000A097C14140000000E0F87F56C0FFFFFF9F04B9414000000080FE8256C0000000A063B54140FFFFFFDFDB8356C0000000A0EAB44140FFFFFF5FBB8456C0000000C097B6414000000040CA8456C0000000C075BC414000000040438556C0000000E035BD414000000080868656C0000000E09DBC414000000020C98856C0000000A0A4B84140FFFFFF1F0D8B56C0000000603CB6414000000080BD8A56C0000000402FB1414000000080F88856C0000000A007B14140000000607A8856C0000000402CB44140000000002F8756C0000000E076B5414000000040738556C0000000408AB54140000000A0D48456C00000002009B44140000000A0938556C000000040D5B0414000000080C28656C000000080D8AE4140FFFFFF1F518656C0000000603CAC414000000060CE8656C0000000A047A84140FFFFFF1F0C8A56C00000000033A7414000000020DD8A56C0000000202BA4414000000060BC8956C0000000A0CCA14140000000C0C78656C0000000C0C5A1414000000040C48556C00000002090A04140000000E0698456C0FFFFFFFF3A9B414000000000B18456C0FFFFFFFF8D984140000000E0FE8356C0000000805D95414000000060218456C000000080DD924140000000A04E8556C0000000A001904140000000E0328956C0FFFFFFFF79914140000000C0868A56C00000002097904140000000006A8B56C000000020E88D4140FFFFFF3FD28A56C000000040F689414000000080868C56C0000000203C85414000000000AD9256C0000000E033864140000000E08A9356C0000000C016804140000000202B9356C0000000E03E7D414000000000E28F56C0000000C091794140FFFFFF3F7B8F56C0000000002E784140FFFFFFBF8A8F56C000000080DA754140000000C0119156C0000000E0C072414000000020F69256C000000000FC704140000000202A9356C0FFFFFFFFB66E4140000000A04C9356C000000000076D4140FFFFFF1FA99456C000000080D56C4140000000E0D99556C000000060276E414000000000DA9956C000000020A76B414000000020079B56C0000000C0876A4140FFFFFF3FBF9B56C0000000E0EC6A4140000000C0619B56C0000000C0B26F4140FFFFFF9F099C56C0000000606E714140000000201D9E56C000000020C2704140000000C0619E56C000000000CA6D414000000000EC9C56C0FFFFFFBFA0694140FFFFFF7FDE9D56C0000000605B664140FFFFFF3FBA9C56C0000000406061414000000040E49C56C0000000A0DF5E414000000060199F56C0000000E0085D41400000006048A056C0000000E06B5D4140FFFFFFFF15A156C0FFFFFF9FCA5F414000000040EB9F56C00000004005624140FFFFFFFF14A056C0FFFFFF3F19654140FFFFFFDF06A156C0000000001E674140000000A0BEA156C00000006056674140000000400EA356C0000000C029654140FFFFFF3F21A256C0000000E04B5B414000000040DEA056C000000060DD59414000000020149E56C000000000255A4140000000A0D69D56C0FFFFFF5F05564140FFFFFF5F90A056C000000020AC514140000000607EA256C0000000C085514140000000000BA356C0000000606E5341400000000080A256C000000020CD574140FFFFFF5FE8A356C000000000A35941400000004013A556C000000060A3524140000000A0A1A556C0000000405C504140000000A0F7A456C000000040684D4140000000A0F5A156C0000000801F474140000000A060A256C0000000C08B454140FFFFFF1F34A456C000000040294441400000006024A556C0000000A09642414000000060C3A556C0000000808D3F414000000000C3A456C0000000E01A3A41400000006010A556C0000000806C374140FFFFFF7FA4A656C0000000E0C93341400000004010AA56C000000020D62E4140000000A019AA56C0000000A03D2A4140000000407AAB56C000000060B3284140000000C01EAC56C000000080FA284140000000C097AB56C0000000E0832E4140000000C0FFAB56C0000000205E304140FFFFFFFF56B056C000000020A32F414000000040C2B056C0000000C0942E4140FFFFFF3FDBAF56C000000000AB2841400000004087B056C000000080B5234140000000C0B8B256C00000000065264140000000609CB356C00000000051264140000000C0B9B456C0000000E07F2341400000000035B556C000000060611D4140FFFFFF1F42B756C0FFFFFF7F0D1C4140FFFFFF7F6BBB56C00000006006204140FFFFFFDFC1BB56C0FFFFFFDF091E4140FFFFFF9FF9BA56C0000000E0371A4140000000C0AAB456C0000000C063184140FFFFFF3FAEB356C00000002045154140000000200CB556C00000004007134140000000E02EB656C0000000C0E21241400000006073BB56C0FFFFFF5FC3174140000000A003BD56C000000060F1134140000000C04DBC56C0000000E01E1041400000000001BA56C0000000E0260D41400000000072B756C000000000EC0C4140000000E0B9B856C00000000036054140FFFFFF7FD4BC56C00000004001044140000000A04BBE56C00000006068014140000000E06ABE56C0000000404FFF40400000002083BD56C0000000C04BFD404000000020BFBD56C0000000C0D5FB40400000000028BF56C000000000FCFA4040FFFFFF3F04C056C0FFFFFF9FF3FB4040000000609CC056C0000000A0C9FE4040000000E0FCC156C0000000602BFE4040FFFFFFDF75C456C0000000C0C7004140000000C0AFC556C0000000C04AFF404000000060D8C456C0FFFFFF1FC1FC4040FFFFFF7F2EC156C0000000E0D8F7404000000080EDC356C0FFFFFF7FFCEE40400000002082C356C0000000A0F8EB4040FFFFFFBFD8C156C0000000E085E840400000004062BF56C00000004042E6404000000080FABE56C00000000086E44040FFFFFF3FB4BF56C000000020C2E24040000000002EC156C0000000E0C9E1404000000020C7C256C0000000C080E24040000000C03FC456C0000000E0ACE4404000000020BFC656C00000006062E3404000000060C4C856C0000000E0D9E340400000002025C956C000000000CBE2404000000000D5C856C0000000A092DC4040000000A040C856C00000004031DB404000000040B1C656C0FFFFFF5FA7DA40400000000099C356C0FFFFFFDF13DC4040000000207CC256C0000000804DDA4040000000E06BC256C0000000A073D74040000000805CC556C0000000A0CFD4404000000040BFC756C0FFFFFF3FB9D640400000008073CA56C0000000C0F1DB4040000000208CCD56C000000060BFDA4040000000E0C4CD56C0000000A069D84040000000C023CD56C0000000C0C0D54040000000C0E1C956C0000000408AD1404000000040A4C956C0FFFFFF5FDECE4040000000E0C0CA56C000000020E3C940400000000005CC56C0000000208FC94040000000C082CE56C0000000E094CB40400000008091CE56C00000008036C74040000000C0A9CD56C00000000007C540400000000010CD56C0000000C0EDC44040000000C0B3CB56C000000020FDC24040000000C08BCB56C0000000A089C14040000000004ACD56C00000004098BC4040000000A089CE56C000000060CFBA404000000000E7CE56C0000000C0C2B8404000000020A2CB56C00000000044B94040000000A0FDCA56C000000000BBBB4040FFFFFFBF25CB56C0000000208FC04040000000008FCA56C00000000082C14040000000403EC856C0000000E020BF4040000000009DC756C0000000401EBC404000000060A9C756C000000060F6B94040000000605AC856C0FFFFFF3FB8B84040000000A0B9CC56C0FFFFFFBF07B640400000000011CD56C0000000C007B54040FFFFFFBFD6CB56C00000008027B2404000000000CFC856C0000000C0C6B14040FFFFFF1F58C656C0000000601BB54040000000C07DC556C0000000E03EBB4040FFFFFF7FB8C456C0000000A0ACBB4040FFFFFF1FE7C356C0000000A0E1BA4040000000E0EFC356C00000002045B74040FFFFFF3F09C556C0000000C080B44040FFFFFF1FD8C656C0000000205BB24040000000E058C856C0000000E000AE40400000006012C956C0000000E043A9404000000020D7C756C0000000605EA240400000000096C656C000000020E69F404000000080E2C456C0000000006CA540400000000072C356C0000000E093A540400000006096C256C00000000016A44040000000A07DC356C0FFFFFFBF759F404000000060E3C556C0000000A0E39C40400000004086C556C000000060AF9440400000008022C656C00000002092924040000000E0C3C756C0000000E0C6904040000000405ECB56C0000000A03C934040FFFFFF1F83CC56C0000000C0FA914040000000C031CC56C0000000207E8E4040FFFFFFFF64C956C0000000809A8B4040000000E089C756C0000000A06488404000000060EFC756C0000000000D8640400000002007CA56C0000000E02C8540400000008048CA56C000000060C7824040FFFFFF5F60CA56C000000040AB814040000000A04BD056C0000000A0B9814040000000A05CDB56C0FFFFFFBFB7814040FFFFFF1F14DD56C0000000A0C6814040000000400D0457C00000008048814040000000A0E42D57C0FFFFFF3F2382404000000020A53E57C0000000405282404000000040DF4E57C00000006076824040FFFFFF3FA65E57C000000000BD82404000000060C06057C000000020B582404000000000D37357C000000060E4824040000000E07A8257C000000020FB82404000000020568257C0000000009AA24040FFFFFF9F4C8257C00000002028C7404000000080EE8357C000000020E2C94040000000C08B8557C000000000BFCA404000000020518657C00000000058C9404000000040EE8957C00000004096C8404000000080358A57C0000000C000CC404000000060248D57C0000000E0E3CA4040FFFFFF1F7F8D57C0000000206CC74040FFFFFF3F108F57C000000060E0C74040FFFFFF3F468E57C0000000E0F8CA4040000000E02E8F57C000000080D4CB404000000020929157C000000000E7C74040000000C0699157C000000060D4CA4040FFFFFFDFDA9157C0000000406FCB4040FFFFFFDF1F9357C0000000A038CA4040000000405A9357C0000000A049C74040FFFFFF3F0A9557C0000000805CC9404000000080BA9757C0000000801AC64040000000004C9957C000000000B8C74040000000E0D39757C0000000004DC9404000000060B89757C00000008086CB404000000060439857C000000020F2CB4040000000C02D9957C00000004098C9404000000040059A57C00000000068C94040000000006C9B57C0FFFFFF1F6FCC4040FFFFFF7F5F9C57C0000000405ACC404000000040E69C57C0000000405BCD4040000000E0EC9B57C0000000C0F4CE404000000000E69B57C00000000077D14040FFFFFFBF7E9E57C0FFFFFF3FE4D04040FFFFFFDFF99D57C0000000A037F84040000000E0889D57C0FFFFFFFF2B194140 Arkansas 05 0.26909722222222222222 0.22028985507246376812 0.24404086265607264472 0.23224852071005917160 0.25079872204472843450 0.27500000000000000000 0.28670360110803324100 0.28456221198156682028 0.26975763962065331928 0.29056603773584905660 0.27697441601779755284 0.25316455696202531646 0.29381443298969072165 0.30872913992297817715 0.35091023226616446955 0.42964352720450281426 0.45012165450121654501 0.43027888446215139442 0.41864406779661016949 0.48788546255506607930 0.45041551246537396122 0.40819277108433734940 0.43283582089552238806 0.42845223195661243221 0.43563547200653861872 0.44472049689440993789 0.46537396121883656510 0.44743543106584212441 0.44270462633451957295 0.48114243866715488832 0.49858757062146892655 0.48017771701982228298 0.49671268902038132807 0.49434673366834170854 0.50953846153846153846 0.52249338429873566598 0.52637454646943901758 0.54052658750645327827 0.52896305125148986889 0.54220121539500337610 0.54033422735712812049 0.55972495088408644401 0.58415094339622641509 0.59943830086010180797 0.63851946803396891524 0.64230148246000293556 0.64304461942257217848 0.65377298668357641091 0.65082644628099173554 0.66554192229038854806 0.64606690365509069365 0.60985609775705442560 0.61767039307609087631 0.60068442595450580420 0.59978479650610798152 0.60068259385665529010 0.60033043756328945265 0.58558738397045613335 0.55647408084037451473 0.54217519890032073978 0.53544985851067953638 0.54267654099341711550 0.56788147265755872390 0.57359874279727606076 0.57411661374231470847 0.58135726450936722128 0.58052399286317964128 0.58084369024856596558 0.56829419035846724351 0.56765993805404250774 0.56600508905852417303 0.54766240157480314961 0.55658837720854324842 0.56945336855381832893 0.59139582055757347045 0.58799351240862938523 0.58508783122793585981 0.56795024739405080868 0.57611718894863712220 0.58318661971830985915 0.59718219053398058252 +31 0106000020E610000006000000010300000001000000020100000000006044FF54C0000000C0997E414000000040910755C0000000207E7E414000000000BA1455C0000000E0977E414000000060A01255C000000020F39A4140000000E0750E55C0000000407CA14140FFFFFF9F800B55C0FFFFFFBFD89E414000000060810655C0000000406E9F4140000000E0BC0255C0000000C0E3A24140000000A0F90155C0FFFFFF5F71A5414000000080DE0155C000000020A3A9414000000060670055C0000000E0B9AF414000000040CF0055C0000000002DB44140000000E019FD54C0000000804CBA4140FFFFFF9F3DFA54C000000040FDBC41400000000066F854C0FFFFFFDF5AC141400000000022F554C0000000A070C2414000000000A8F154C0000000C0BBC641400000006010EB54C00000000051C841400000008049E754C0000000C02EC9414000000000E8E354C00000000010C74140000000005FE054C0000000609FC74140000000A054DD54C0000000A073CC414000000020C6D854C00000002007D0414000000020F4D554C0000000009ED341400000004019D354C00000008002D44140FFFFFF7F95D054C00000000073D841400000002036D054C0000000C0B0D94140000000409BCF54C000000080EEDB4140000000C0DCCB54C0000000204CDD4140000000C031C954C0000000A09FE14140FFFFFF5F90C754C000000080C4E1414000000060D6C354C0000000802BE44140000000202BBF54C00000002012E34140000000C09DBD54C0000000605BE541400000006009BA54C0000000C0A4EF41400000002080BA54C000000060C4F64140000000A057B954C0000000E063F9414000000020CDB654C00000004045F9414000000040C1B354C000000020DFF5414000000040B1B154C00000000071FA4140000000A082B154C00000008066FE4140000000C0D8B054C0FFFFFF1FF1FF4140FFFFFF7F35A954C0000000E09E064240000000E031A854C0000000A0F406424000000080AEA654C0000000A08005424000000020E7A554C0000000A0DF024240FFFFFF7FC5A654C0FFFFFF3F74004240000000005BA654C0000000604DFB41400000002077A354C0000000C061FA4140FFFFFF9F71A054C0FFFFFF3F7CFC4140FFFFFF9F699E54C00000006020FF4140000000A0239A54C0FFFFFFFFA309424000000040ED9754C000000020A20C4240FFFFFF9FF69354C0000000A0A20F4240000000A0C99054C000000080680F4240000000804B8D54C000000060D112424000000000DC8954C0FFFFFF1FDF114240000000A08E8754C000000020520C4240FFFFFFFFF98454C000000060D10C4240000000204F8154C0000000609A10424000000020B87B54C000000080B4214240FFFFFF5F507A54C00000006037254240FFFFFF1F287554C000000000D62A424000000060C06E54C0000000E0272A4240000000C0676D54C0FFFFFF7FB72A424000000060626F54C0FFFFFF5F512E4240000000C06D6F54C00000000026324240000000A0B06C54C000000040633C4240000000A0FA6C54C0FFFFFFBF7942424000000040E16A54C0000000A0794B424000000060195654C0FFFFFF9F5349424000000000D23954C0000000E05848424000000060A43554C0000000C01E484240000000801B2754C0FFFFFF7F55474240FFFFFF1FDC1B54C0000000A08846424000000000140354C00000008008464240000000208A0154C000000060C3454240000000A0EAED53C00000004021464240FFFFFFBFA8E053C0FFFFFF9F1946424000000060E4CD53C0000000405F464240000000A03CC953C000000040E545424000000020FDB253C00000008092454240FFFFFF5F31AF53C0000000C0E5454240000000205D9D53C0FFFFFF3F4F454240000000608F9453C000000000D4454240FFFFFF7F4E8353C000000080B7464240000000E0867953C0000000E0C646424000000040E47053C000000020D7464240000000C07B5453C0000000C0E6464240000000C0594B53C0000000603447424000000000253B53C000000040EE46424000000000FC3A53C0000000A0EE464240000000C0112453C0FFFFFF7F1247424000000000D81F53C0000000E024474240000000E0221553C0000000E02C47424000000040270853C0000000205147424000000000F10253C0000000404A474240FFFFFF1F200253C000000000D741424000000000D40553C0000000E0744042400000000078FE52C0000000E0D5374240000000A010FE52C0000000A022354240000000601A0053C0000000009F35424000000060E1FC52C0000000C0C72E42400000006066FB52C0000000E02C364240000000802CFB52C0000000E0EB2C42400000002034F352C0000000C0700E42400000006020F352C0000000205209424000000000BBF652C0FFFFFF7F860D4240000000207BFA52C0000000A0551F4240000000E047FD52C0000000C035214240000000A048FC52C0FFFFFFDFAC25424000000000930053C000000080E82842400000008042FD52C000000080CD18424000000040CEFE52C0000000A0B915424000000000B90B53C0000000C05928424000000080020E53C0FFFFFF3FF7254240000000A0360753C0000000605316424000000080150953C0FFFFFF7FE7124240000000E0090F53C000000000E9144240000000A0211353C0000000E06B1B4240000000E0A11153C0000000A0200E424000000000C11E53C0000000A0360A424000000040E81A53C0000000608007424000000000792153C0000000E0EA00424000000020042653C0FFFFFFFF4B01424000000080892953C00000002069084240FFFFFF7F862853C000000040C0044240000000802B2C53C0000000A059064240000000207F2E53C0000000C01214424000000040142C53C0000000C0B2254240FFFFFF7FB43153C0FFFFFFBFDD2D424000000020243B53C0FFFFFF9F3B32424000000040722F53C0FFFFFF5F57284240000000A0472D53C0000000A01022424000000020AF2F53C000000000F11D4240000000E0AE3053C00000000082124240000000A0662C53C0000000A019FF4140000000A06E2F53C000000060E3F74140000000409F2C53C00000008084F84140000000C0341A53C0000000A01DFD414000000060C61753C00000000057F7414000000060AE0D53C0000000400AFD4140000000A0BC0553C0FFFFFFBF40FB4140000000C0AA0153C000000080D0F54140000000C0DB0453C0000000E0DEE04140000000A0C10253C00000006088D7414000000020B50A53C00000002032D9414000000020B40653C000000020F2D44140000000805B0153C0000000E0A4D541400000000039FF52C00000004044F24140000000005BF452C0000000E035F6414000000000EFEF52C00000008046EF4140FFFFFFFFADEE52C00000006024D54140FFFFFFFFDBF152C0FFFFFF5F12CA4140000000400EF952C000000060CDD0414000000000640353C0000000C011B54140000000A09F0B53C000000060B7AB414000000060EC1F53C0000000A046B5414000000000292253C000000020A0B9414000000000571D53C00000000091C04140000000E0931C53C0FFFFFFBF82C6414000000060362153C000000040F0C94140000000C0851F53C00000004059C5414000000040D82853C0000000609AC24140000000C03D2853C0000000000DB84140FFFFFFDF242D53C000000040BAB4414000000020A44653C0000000806CC6414000000080EC3E53C000000040DFB7414000000040792C53C0000000C0DAAC414000000040542753C000000000EFA2414000000060D62553C000000040F2A74140000000609F2253C000000040DFA64140000000C0742053C0FFFFFF9FD79F414000000040F72853C0000000201596414000000000BC2653C000000060BC91414000000040F32753C0000000A03C88414000000080642B53C00000004015834140000000A05D3653C000000060B97D414000000060413C53C0000000C082834140FFFFFF1F5E3C53C0000000E0F6884140FFFFFF7F104353C0000000C0CB8B414000000000B34653C000000020438B4140000000E0134753C0000000E073884140FFFFFF7F134053C000000000C586414000000060703A53C000000040DE77414000000040692953C0000000C00274414000000040E22A53C0000000E0287C4140FFFFFFDFE72153C0000000E0C07C4140000000607B1C53C0000000E021824140000000C0231B53C0000000A01E794140000000004C1753C0000000E0B778414000000000151553C000000020EC7C414000000040241453C0000000C072794140000000E0121E53C0000000E07C644140FFFFFF1F052853C0FFFFFFDF205C4140FFFFFFDF3B4353C0000000207659414000000080884953C0000000A0D561414000000000484853C000000040AB574140000000C0054A53C0000000C08F54414000000000796253C0000000407F3A4140000000808D6553C0FFFFFF3FE1354140000000C0056753C000000080AF37414000000040916653C000000000D1344140000000A0087053C0000000C00B27414000000020527753C0000000A0AB184140FFFFFFDF3D7953C0000000E0DA08414000000000507B53C0FFFFFFBFF0074140000000A07C7D53C0FFFFFF1F3918414000000080587D53C0000000E00CFF4040000000403A8253C00000008007F54040000000201AA553C0FFFFFF1FE5F0404000000040F7A953C0FFFFFF1F6EF94040000000A0C4C453C000000040FC264140000000009EDC53C0000000603D4F4140000000A02EDD53C0000000E02951414000000060B8EA53C0000000207C664140000000C0E6EB53C0000000A01267414000000080E0FA53C0FFFFFFDF6967414000000060D11454C0000000204A68414000000060F02354C00000006059684140FFFFFF3F343354C0000000607668414000000020483254C0000000E06678414000000040C53554C0000000204280414000000000433954C000000060A587414000000000613B54C0000000C0F58C4140000000808D4254C000000000C384414000000000324454C0000000808288414000000000D24154C0000000007E8D414000000080234354C0000000C0E5904140000000C0274354C0000000606593414000000020A95454C000000000F6944140000000E02D5754C000000060D8944140000000A0FB7054C0000000405B974140000000C0B77754C00000006070974140FFFFFF1F2C7E54C0000000A019984140FFFFFF7F738D54C000000060B898414000000020D09154C0FFFFFFFFF5984140FFFFFF7F879454C0000000809397414000000080749654C0FFFFFF7FA9984140000000400C9754C0000000E06A974140000000A0C49754C0000000806397414000000060EF9854C0FFFFFF9FA79A4140000000E0069C54C000000020B4954140FFFFFF1FDF9D54C00000004035964140000000A093A154C0000000A0C8934140000000607BA454C0000000802293414000000040E3A954C0000000604A8F414000000040E8AB54C0000000208C8F4140000000C008AC54C0000000C0848C414000000080A1AC54C000000080AD8B4140000000C05DB154C0FFFFFF1FEF8A414000000000D0B854C000000060168741400000004077C054C0000000201983414000000060CDC654C0000000200980414000000020D5E054C0000000A0FA7E4140000000A029E354C000000020A97E41400000002008FC54C000000060A47E41400000006044FF54C0000000C0997E41400103000000010000000700000000000020BD0153C0000000804247424000000020EAFF52C00000006040474240FFFFFFDF5CFA52C0000000A0714542400000000030FB52C000000040AB3C4240000000C08BFE52C0000000A02F3D4240000000807BFE52C0000000604B42424000000020BD0153C000000080424742400103000000010000000800000000000020BAF952C00000008031474240000000E033F852C0000000E026474240000000607BF152C000000040581D4240000000C0E0E252C000000000E9E44140000000A0F9EC52C0000000E061064240000000606AEF52C00000000071064240000000801FF252C000000020D31C424000000020BAF952C00000008031474240010300000001000000090000000000002070DF52C0000000E0D2D54140000000E026E252C0000000E069E24140000000203FDD52C0000000E006CF4140000000E0AEE152C0000000602C9D414000000060F4EF52C0000000404C984140000000A042EC52C000000020149E41400000000060E152C00000008003A44140000000406DDE52C00000006041C841400000002070DF52C0000000E0D2D5414001030000000100000008000000000000A0160153C0FFFFFF3FE3884140000000C071FE52C000000020DF8E4140000000E0A6F652C0000000806995414000000080E0F052C0000000C09C98414000000040F0F352C0000000A0FE94414000000040DAF952C000000080F5904140000000202F0053C0FFFFFF3FE3884140000000A0160153C0FFFFFF3FE38841400103000000010000000900000000000060D52253C0000000003E4B4140FFFFFF7F872353C0000000202A4E4140000000806A2253C0FFFFFFFF944E4140000000C0011F53C0000000004C594140000000C0AA1B53C0000000205E614140000000C0FA1753C0000000200C684140000000C0691253C0000000E04170414000000060081E53C000000080BD58414000000060D52253C0000000003E4B4140 North Carolina 37 0.28819444444444444444 0.28212560386473429952 0.28149829738933030647 0.27662721893491124260 0.33226837060702875399 0.37205882352941176471 0.37534626038781163435 0.34216589861751152074 0.34141201264488935722 0.37106918238993710692 0.35038932146829810901 0.31548198636806231743 0.36254295532646048110 0.36713735558408215661 0.43502824858757062147 0.47904940587867417136 0.50060827250608272506 0.49288560045532157086 0.50847457627118644068 0.55231277533039647577 0.53684210526315789474 0.51903614457831325301 0.53912256897331524197 0.51314142678347934919 0.52063751532488761749 0.53540372670807453416 0.54135338345864661654 0.52237177155329210622 0.50676156583629893238 0.55108019040644452581 0.55826271186440677966 0.55673274094326725906 0.55522682445759368836 0.56438442211055276382 0.57753846153846153846 0.59070861511320199941 0.59810214903711973207 0.60996386164171399071 0.60190703218116805721 0.61985145172180958812 0.62946152259129358366 0.64538310412573673870 0.66226415094339622642 0.68439529576970335264 0.69940714629065854831 0.69616908850726552180 0.69609062025141594143 0.70818008877615726062 0.69536271808999081726 0.69325153374233128834 0.68006562756357670221 0.66299541763807380014 0.66238730616660656329 0.65020465678051399047 0.66333312234951579214 0.67053469852104664391 0.67414592549165911635 0.67092524204012376485 0.65425896323361498059 0.64402049402257674845 0.64112106058844051634 0.64957360861759425494 0.66556229758403752373 0.66771433560328269600 0.67704884805080737788 0.68554303681383466527 0.68669984662096597490 0.68534894837476099426 0.67951455219687605349 0.67964327672754459041 0.66165394402035623410 0.66914370078740157480 0.65398897797961162752 0.65981295066752338117 0.66531440162271805274 0.66140105312270879158 0.66371060354727574327 0.63935227739424637710 0.62981642870387539403 0.62976819248826291080 0.63645327669902912621 +11 0106000020E6100000010000000103000000010000004601000000000080940456C0FFFFFF1F68C14240FFFFFFDF9F0556C0FFFFFF7FF6BC424000000000F31356C000000060AFB84240000000C0FC1656C00000004064B44240000000E0DE1A56C000000020CCB54240000000E0ED1D56C0000000004CB3424000000080B92056C000000040FFA5424000000060172056C000000000FFA04240FFFFFF3FD81C56C000000060539A4240000000800A1B56C0000000A01594424000000080D41C56C000000040A18C4240FFFFFFDF831E56C0000000003C894240FFFFFF9F671F56C000000020BA884240FFFFFFFF1A2156C0000000604A884240FFFFFF1FCB2356C00000000052894240000000604F2756C000000040F58D4240000000400E2C56C00000002055914240FFFFFF9F4D2F56C00000004012924240000000C0C62F56C0000000407893424000000020403756C000000080E199424000000020AE3B56C0000000C0F49B424000000020903F56C0000000202A9C424000000080294456C000000040CA974240FFFFFFFF794756C0000000805A8E4240000000C05D4956C0FFFFFF7FED8B4240000000E0D94A56C0000000E03888424000000040284B56C0000000804A834240000000A09D4956C0000000E0CC7F4240FFFFFF9F4F4856C0000000807A7E424000000080624C56C0000000804E7E4240FFFFFF7F714D56C000000060B583424000000020364F56C0000000805785424000000040E65056C0000000E0268B4240000000E0305256C0000000E0AD8B424000000020695356C0000000E0ED8A424000000020D25356C0000000E0CB87424000000060E95056C0000000C08C834240000000A0C45056C0000000A01C814240000000E0185256C000000000E67F424000000020E75356C0000000403D81424000000040825856C0000000A04C864240000000C0515856C0000000C0AE8C4240000000801F5B56C0000000E08F914240FFFFFF7F315C56C0000000202995424000000040F75D56C0FFFFFFBFB49C4240FFFFFF9FC75D56C0000000407AA0424000000080555F56C0000000A0C4A0424000000080E36056C0FFFFFF1F61A3424000000080E36056C00000000009A7424000000080096056C0FFFFFF1F2BAA4240000000E0FF5D56C0000000C071AB424000000020E35B56C00000002088AD4240000000605D5B56C0000000409CB4424000000020085D56C00000000002BA424000000080AA5F56C0000000E0F0BE424000000020996156C0000000E035C9424000000000DB6056C0FFFFFFBFD6CE4240000000403A6156C0000000803FD3424000000020DB6056C00000000005D74240000000A0606156C000000020EFD84240FFFFFF3F366556C0000000A061DA424000000040A76A56C0000000006BDF424000000040416B56C00000002059E44240000000403A6C56C00000008003E74240000000E09E6E56C0000000A0A5EB424000000080827656C000000020D9F34240FFFFFF5F1B7756C000000000E7F34240FFFFFFDF797756C00000000029F24240FFFFFF9FA27956C0000000A01DF0424000000020067C56C0000000C063F0424000000080A67E56C0000000A0B8F44240000000A0537D56C00000006058FB424000000020B18056C0000000A012FC4240000000E0AE8256C0FFFFFF5F21FF424000000040A38756C00000008021044340000000209F8856C0000000E0E706434000000020488D56C000000040610B434000000080429056C000000040A30F434000000060899256C0000000405A154340000000C08C9556C0000000C02718434000000060589756C000000080FD1D434000000060A39756C0000000606A294340FFFFFFBFF49656C0FFFFFF1FC32E434000000020BC9556C00000004007324340FFFFFF5F519356C0000000A0B3364340000000A0029156C0000000606442434000000000B89056C0000000C031444340000000A06B8F56C0000000000A484340000000E0C18B56C0000000601D4E4340000000C0BF8B56C0000000A052544340FFFFFF7FF18C56C000000080A5594340000000A0948C56C0000000E0AA5C434000000020758A56C0000000E0F4624340000000C0A68856C0000000C08A64434000000060CA8756C00000002077664340000000603D8756C0000000C04C6A434000000000808856C0FFFFFF1F306D4340000000809C8F56C0000000A00E75434000000000DA9156C0000000205D764340000000A0769456C00000006063764340000000C06F9A56C0000000A02D7B4340000000E0119E56C000000060C67A434000000080F2A156C000000040207243400000004080A456C0000000A0876F43400000004024A856C0000000E0BD704340FFFFFFDFCEAA56C000000060B67743400000004030AD56C000000060D6844340FFFFFF1F49AD56C00000006072874340000000802FAC56C000000060FE8B434000000000DFAD56C00000008075924340FFFFFFDFF6AD56C00000006012994340000000A0DEAE56C0FFFFFF7FC49C4340000000C03CAF56C000000040B89F4340000000C0E0B156C0000000A0FDA54340000000806EB656C0000000A0DBAC434000000040AABC56C00000006046B343400000006053C256C0FFFFFF7FE2B84340000000E01EC456C000000080ABBC4340000000C0FDC556C0000000E0B3C3434000000000FFC956C000000060BBC643400000000002CD56C000000080CDCC4340000000A054D456C000000020CCD74340000000607ED756C000000000C1DC434000000020E6D756C00000006071E14340000000006ED856C000000000E2E64340FFFFFF7FBFDC56C00000006078EE434000000000DDDC56C0000000A04FF1434000000080C7DB56C0000000206FF34340000000808BDB56C0000000C0FEF54340000000A09FDC56C0000000A018F94340000000C02FDF56C000000080BC004440000000A041E056C0000000008A084440FFFFFF3F08E156C0000000C038114440000000406BE056C0000000A0A819444000000080EEDF56C0000000202D2044400000000026DF56C0000000C0A1274440000000C0B5DC56C0000000809A2F4440000000E0CDDA56C0FFFFFF1F85314440FFFFFF3FB0D856C0000000E03832444000000040DBD756C0000000209533444000000060AAD856C0FFFFFF7F3F394440000000A0FCD756C0000000C0774044400000006074D856C0000000C0A5434440000000806CDA56C0FFFFFF9F24464440000000C04FDA56C00000002057494440000000000AD856C0000000803D4D4440000000A0C5D056C0000000A0DC514440FFFFFF1FC1CD56C0000000A068524440FFFFFF5F66CA56C000000000025444400000002044C856C0000000A050574440000000E0ADC756C0000000A04A5A4440000000A0EFC556C0000000607A614440000000A0B0C556C0000000A0B76A4440FFFFFF3F26C356C0000000409670444000000000EEBE56C00000004043764440000000407CBD56C000000020AA7944400000000019BD56C0000000A001894440FFFFFF5F4CBD56C0000000A05B8D4440FFFFFFBF61BF56C0000000C07A924440000000202BC156C0000000C039954440000000C09AC356C0000000A08F964440000000607FC656C000000080A29D4440000000E08CC656C0000000E047A24440000000A0B0C456C0000000E0DDAA44400000000092C356C00000006060B3444000000060C2C156C00000008035B64440000000600BC056C0000000C02DB7444000000020C7BC56C000000000EBB544400000006006B656C000000060E9B84440000000E0E9B156C0000000C093B944400000006053AD56C0000000A09BB94440000000A029AA56C0000000402DBB4440000000E071A656C0000000203AC14440000000209DA256C00000000053C34440000000A01E9D56C0000000A086C34440FFFFFF7FD69B56C00000000094C54440FFFFFF7F129B56C0000000609CC84440000000A04B9656C0000000E01DCB4440FFFFFFDFB79556C00000008028CD4440FFFFFF1FD59556C00000006015D34440FFFFFF9FDD9456C0000000A082DC444000000040839356C0000000E0D3E0444000000000579056C00000000010E44440000000A0888C56C0000000802FE74440000000A0E38956C0000000A023F7444000000080218956C000000080F2FD4440FFFFFF5FA28956C0000000604704454000000020C28A56C000000040D007454000000060AA8A56C000000080470D454000000000458B56C0FFFFFF9F6C0F4540000000C0428C56C000000040B40F4540000000A0C78E56C0000000C071144540000000E0B59456C0000000C041194540000000E0889756C000000020E81A4540000000200F9A56C0000000000F1F454000000040C09A56C0FFFFFF3FC8214540000000205F9B56C0000000E0992B454000000020439C56C0000000E0162E4540000000406D9F56C0000000A0C3314540000000C011A456C0000000C0FE354540000000E0C5A656C0000000A0F33A4540000000807EA956C0000000E0E13C4540FFFFFF9FB6A956C000000040523F454000000060DAA856C0000000C032414540000000E0E09A56C00000000012414540000000C01B7B56C0000000A086404540000000606A7556C0FFFFFF5F71404540000000C0A15956C0000000C0AD3F4540FFFFFF1F015756C000000060BB3F4540000000E0193C56C0000000A0D43E4540FFFFFFFFF43056C000000000D63E4540000000A0372D56C000000000AD3E4540000000C0101356C0FFFFFFDFF83E454000000000760C56C0000000A0AB3E45400000002007F355C0000000E09B3E45400000008090F555C00000002038284540000000C0A7F055C0000000C00614454000000040EAEA55C000000040A80745400000004035E755C00000006075EC4440FFFFFF3FE9E155C0FFFFFF9F9EDC4440000000E016E255C0000000A01FBC4440000000A013E255C00000002091A64440000000E007E255C0000000A03D964440000000A00CE255C000000060458144400000004017E255C0FFFFFF9F695F44400000000061E255C0FFFFFF5F4F3F44400000008048E255C000000000DB3D44400000000043E255C0000000E045154440000000204AE255C00000002093F14340000000E046E255C0000000E0FECD4340000000E077E255C0000000001DBD4340000000E092E255C000000000DEAC4340FFFFFF1F40E655C0000000604CAB4340000000E003E855C0FFFFFFFF58A743400000006014E755C0000000C019A643400000004069E755C00000008005A4434000000060D7E655C0000000800BA143400000008069E555C000000020D79F434000000080ABE555C000000000AF9A43400000008007E655C0000000405C994340FFFFFF3FE8E655C0000000C018994340000000803BE955C0000000A091954340FFFFFF9FE6EA55C000000060C69243400000008034EA55C0FFFFFF3FB99043400000008062EA55C000000020868E4340000000406DE855C0000000004E8D43400000002060E855C000000080638B4340000000202BE755C000000060D48A4340000000E075E555C0000000E0FD874340000000603BE555C0FFFFFF7F747F434000000000E1E555C0000000203E7F4340000000E010E355C0000000E0107D43400000006024E255C0FFFFFF9F5A7B434000000080EEE155C00000002049774340FFFFFF3F82E255C000000080D2734340000000A0C7E355C000000000566F4340000000803BE355C000000060CF6D43400000004081E055C0000000E0D4654340000000C037E155C0000000E06A634340FFFFFF1F83E055C000000040866243400000004088E055C0000000004A5E4340FFFFFF1FCFE255C0FFFFFFFFCD574340000000A0A9E555C0000000A0095643400000002003E855C0FFFFFF9F47524340FFFFFFBF3BE855C0000000C0BB4F434000000040ABE755C0000000E0B24C434000000080FFE855C000000040ED4B4340FFFFFF5FC8E955C0FFFFFF9F744943400000008011EB55C00000000012464340FFFFFF5FB0E955C0000000A0F741434000000080D3E955C0000000800E404340000000A083EB55C0FFFFFF3F834043400000002057EC55C0000000E0A23D4340000000E063F055C0FFFFFFFFA93B4340000000E08DF055C000000020823A4340000000004BEF55C0FFFFFF7F0539434000000040E6EF55C0000000E07F354340FFFFFF5F2DF255C000000060663043400000008068F555C0000000801F2D4340000000C067F655C0000000E09E244340000000803BF755C0FFFFFFBF8624434000000040F0F755C0FFFFFF7F8C284340000000608AF855C00000000064284340000000A0DCF855C0000000007C264340000000C080FA55C0FFFFFF5FF92343400000004079FA55C000000040B32643400000004042FB55C0000000C002274340FFFFFF9FB8FE55C0000000E0DB1E4340000000C01AFF55C0000000600E1E43400000006096FE55C0FFFFFFFFB0194340000000A0AAFB55C0000000A0E7154340FFFFFFBFA5FB55C0000000E02914434000000020D6FC55C00000006086114340000000E04DFE55C000000080DD104340000000E02F0156C000000000390D434000000000CA0056C000000000D20B434000000060C0FD55C000000040620C4340000000406BFE55C0000000206209434000000000390256C000000040EC06434000000000C20256C000000080C605434000000080A70256C000000020E704434000000080630156C0000000C04A044340000000A0DE0156C0000000E00D014340000000A0630156C0FFFFFF9FCEFC424000000080B80256C0000000E066FA4240FFFFFF5FAC0256C0000000A09DF74240FFFFFFBF220456C00000002003F74240000000600D0556C000000000D5F8424000000040600556C0000000803AF64240000000C0F20156C0000000A073F54240000000A0B30156C0000000E0EFF34240FFFFFF1FDF0256C000000040B0F24240FFFFFFBF670656C0FFFFFF5FFDF34240000000407E0656C00000006099F24240000000E0D80456C00000006014EF4240FFFFFFFF300256C0000000E0FFEB4240FFFFFF5FB20256C000000040ECE9424000000080B60556C00000006066EA424000000080810556C000000080A7E84240FFFFFFDF460256C0000000A020E7424000000060A30456C0FFFFFF9F21DE4240000000808D0856C000000000B2D9424000000020330A56C00000006091D44240000000A0160A56C00000000072D0424000000040960856C000000080B2CA424000000080940456C0FFFFFF1F68C14240 Illinois 17 0.82291666666666666667 0.77971014492753623188 0.76163450624290578888 0.71893491124260355030 0.69808306709265175719 0.74264705882352941176 0.79362880886426592798 0.74884792626728110599 0.77028451001053740780 0.81509433962264150943 0.78309232480533926585 0.73125608568646543330 0.76632302405498281787 0.66495507060333761232 0.78970495919648462021 0.86679174484052532833 0.89111922141119221411 0.87307911212293682413 0.92598870056497175141 1.00000000000000000000 0.93351800554016620499 0.88240963855421686747 0.91632745364088647671 0.87234042553191489362 0.90273804658765835717 0.90144927536231884058 0.89829837752275425406 0.89196071298654056020 0.89750889679715302491 0.91944342731600146466 0.93290960451977401130 0.92276144907723855092 0.91683103221564760026 0.91143216080402010050 0.91723076923076923077 0.92061158482799176713 0.93776165224672062517 0.92901393908105317501 0.90226460071513706794 0.91042088678820616700 0.89828760057767691355 0.89980353634577603143 0.91962264150943396226 0.92434614709496226084 0.94584201249799711585 0.94877440187876119184 0.96505042132891283326 0.96677235256816740647 0.96246556473829201102 0.94856850715746421268 0.93364324127244553824 0.89050566765817187877 0.88351965380454381536 0.85694155539153190633 0.84112918539148047345 0.83515358361774744027 0.82652027927303736076 0.81265595368799281365 0.78963233614980589176 0.76898404631982338485 0.76109625150211264876 0.77633153800119688809 0.79365670252764024867 0.79497118910424305919 0.79001418823052496453 0.80047163631599633172 0.80267317745015181394 0.80679373804971319312 0.79635352286773794808 0.80033109046245861369 0.79249363867684478372 0.79377460629921259843 0.77598807918824948556 0.79305585302586801837 0.80675975281852917590 0.79093070274833922104 0.78314406022712772745 0.76701934171669958735 0.76465788985722232524 0.77114509976525821596 0.77618704490291262136 +40 0106000020E610000001000000010300000001000000FA000000000000E019FD54C0000000804CBA414000000040CF0055C0000000002DB4414000000060670055C0000000E0B9AF414000000080DE0155C000000020A3A94140000000A0F90155C0FFFFFF5F71A54140000000E0BC0255C0000000C0E3A2414000000060810655C0000000406E9F4140FFFFFF9F800B55C0FFFFFFBFD89E4140000000E0750E55C0000000407CA1414000000060A01255C000000020F39A414000000000BA1455C0000000E0977E4140000000A0992755C0FFFFFF1F8D7E4140000000405B3155C0000000A0CE7E4140000000003B3355C0000000A0127F4140FFFFFF3FF23D55C0000000C00D7F414000000020433E55C0000000000C7F414000000000235155C0000000009A7E4140000000000A5755C000000020B57E414000000040EA5D55C0000000E0B97E4140000000C0F96655C000000020BB7E414000000060A77755C0000000C0037F4140000000C06C9355C000000080687F414000000040099455C000000040627F41400000006012B255C0000000609D7F4140000000C056B555C0000000C0C37F4140000000E048CD55C000000020028141400000008041CE55C000000000EE80414000000000E6E655C0000000E056814140000000601BFF55C0000000A00A824140000000A0790C56C0FFFFFFFFB8814140000000C0690C56C0FFFFFF1F8F80414000000000821656C0000000C07A804140000000C0841856C000000060A2804140FFFFFF3F3D3256C00000006065804140000000E0FF3356C0000000004D80414000000080644056C0000000C004804140FFFFFF1FAD4C56C0000000001A80414000000040E85556C0000000A0F67F4140000000E05F6956C00000000015804140FFFFFF5FE66D56C0000000C0E47F4140000000E08A9356C0000000C01680414000000000AD9256C0000000E03386414000000080868C56C0000000203C854140FFFFFF3FD28A56C000000040F6894140000000006A8B56C000000020E88D4140000000C0868A56C00000002097904140000000E0328956C0FFFFFFFF79914140000000A04E8556C0000000A00190414000000060218456C000000080DD924140000000E0FE8356C0000000805D95414000000000B18456C0FFFFFFFF8D984140000000E0698456C0FFFFFFFF3A9B414000000040C48556C00000002090A04140000000C0C78656C0000000C0C5A1414000000060BC8956C0000000A0CCA1414000000020DD8A56C0000000202BA44140FFFFFF1F0C8A56C00000000033A7414000000060CE8656C0000000A047A84140FFFFFF1F518656C0000000603CAC414000000080C28656C000000080D8AE4140000000A0938556C000000040D5B04140000000A0D48456C00000002009B4414000000040738556C0000000408AB54140000000002F8756C0000000E076B54140000000607A8856C0000000402CB4414000000080F88856C0000000A007B1414000000080BD8A56C0000000402FB14140FFFFFF1F0D8B56C0000000603CB6414000000020C98856C0000000A0A4B8414000000080868656C0000000E09DBC414000000040438556C0000000E035BD414000000040CA8456C0000000C075BC4140FFFFFF5FBB8456C0000000C097B64140FFFFFFDFDB8356C0000000A0EAB4414000000080FE8256C0000000A063B54140000000E0F87F56C0FFFFFF9F04B9414000000020AD8256C0000000A097C14140000000209E8256C0000000007CC54140000000801D8256C0FFFFFF7FB5C6414000000060557F56C000000000E5C74140000000E0957D56C00000002022C44140000000A0A47C56C00000008071C3414000000060987B56C0000000E0BDC34140FFFFFF7FFC7A56C000000040E5C5414000000060507D56C0000000600FCA414000000040407D56C00000008032CD4140FFFFFFFF277856C00000002011D1414000000020497756C0FFFFFF7F9BD0414000000040597656C0000000A096D2414000000020DD7656C000000060E5D54140000000205F7756C0000000E02ED64140000000802D7956C0000000E0F6D34140000000E0807B56C0FFFFFFDF8FD6414000000020EE7C56C0000000C032DB4140000000C0DD7C56C000000080FCDD4140FFFFFFDF397A56C0000000809EE0414000000020087756C0000000C0C4DF414000000040EE7456C0000000000FE14140000000A0317356C0000000C019E34140000000A0957256C0000000601CE7414000000080A07056C000000040A1E84140000000A0196F56C0000000004DE7414000000060DA6C56C000000000ECE9414000000060E46C56C0000000E0C7EB4140000000607E7056C0000000A08AEF4140000000A00A7156C0000000402AF14140FFFFFF7FD37056C0000000C0CAF24140000000003B6F56C0000000201FF5414000000060BD6D56C0000000A0A9F44140000000A0896A56C0FFFFFFDF5CF14140000000C08E6956C00000000078F24140000000404E6956C000000060F7F4414000000020826A56C0000000800AF8414000000000A46D56C0FFFFFF7FB0FB414000000040316E56C000000000FCFF414000000000166C56C0000000404D03424000000020676B56C0000000C09E0A424000000080B66A56C000000060B60C424000000020B96556C0000000009D104240000000A0B96556C0000000607513424000000080966756C0000000E08417424000000080506B56C000000060451C4240000000A0856C56C000000060D21E424000000060736C56C00000000046204240000000E0EA6A56C000000060A0204240000000A08E6756C0000000C0D51E4240FFFFFF5FAA6256C000000080EE204240000000A0436256C0FFFFFF5FDC21424000000000B46256C000000060F323424000000040D56656C0000000C06D274240000000E0DB6756C000000020DA2A4240000000A0C36656C000000080682D4240FFFFFFFFD96256C0000000A0402C4240000000C03C6156C0FFFFFF1F8F2D4240FFFFFFBF476156C0000000E05533424000000040E46256C00000006071384240FFFFFF7F066156C0FFFFFF3F643C4240FFFFFFDF1F6256C000000000C23F4240000000E0735E56C000000060D03F4240FFFFFFBF7C5F56C0FFFFFF3F943B424000000020225E56C0000000001539424000000080B45C56C0000000A06A3A4240000000A08A5A56C0000000C055404240000000A02E5656C00000008053404240000000E0233556C000000040F93F424000000020E23456C000000000FB3F4240000000E0E13356C0FFFFFFBFDE3F424000000000CF2056C000000040EF3F424000000020BE1F56C000000060C33F424000000020BC0256C0000000E08D3F4240000000403E0256C0FFFFFFFFE1444240FFFFFFBFA00256C0000000E0944A424000000060900456C0FFFFFF1FFE564240FFFFFF5FB9F755C000000000AE55424000000000A0F655C0000000C01B5242400000008062EC55C0FFFFFFFF7C5242400000004000E955C0000000E094524240FFFFFFBF2ED655C0000000001A5342400000008036C755C0000000A05C534240000000205DC455C0000000604C5342400000008050B155C0000000A076534240000000E0AEA055C000000020D8534240000000C0969A55C0000000605053424000000080BC8C55C0FFFFFFDF55524240000000A0C27E55C00000002008514240000000A0457255C00000006035504240FFFFFF7FFE5B55C0000000201F4F424000000060355355C0000000002250424000000040715155C00000002012504240000000A0E73F55C0000000207A4F4240000000A0A13255C0FFFFFF9F7C4D4240000000200B3255C0000000C0704D424000000020701055C0000000A0364C4240000000A04A1055C0FFFFFF1F354C4240000000C06F0055C0000000E0C64B424000000020E2FB54C0000000C0AC4B42400000004086EC54C000000000C64A42400000008037EB54C0000000A09F4C424000000020B7DD54C000000000A44C4240FFFFFFBF9BD154C000000040D94C424000000040E7CF54C000000020804B42400000008081CD54C0FFFFFF9F434B4240000000E027BF54C000000060AC4B4240FFFFFF5F65B654C000000020A44B4240000000001AA754C000000080B44B424000000000029354C0000000C0BC4B424000000020E08D54C000000020074C424000000080E08954C0000000602A4C4240000000407C7B54C000000060444C4240FFFFFFDFC77A54C000000000874E4240000000400F7554C000000000454E4240FFFFFF7FC16954C000000060C44D424000000040E16A54C0000000A0794B4240000000A0FA6C54C0FFFFFFBF79424240000000A0B06C54C000000040633C4240000000C06D6F54C0000000002632424000000060626F54C0FFFFFF5F512E4240000000C0676D54C0FFFFFF7FB72A424000000060C06E54C0000000E0272A4240FFFFFF1F287554C000000000D62A4240FFFFFF5F507A54C0000000603725424000000020B87B54C000000080B4214240000000204F8154C0000000609A104240FFFFFFFFF98454C000000060D10C4240000000A08E8754C000000020520C424000000000DC8954C0FFFFFF1FDF114240000000804B8D54C000000060D1124240000000A0C99054C000000080680F4240FFFFFF9FF69354C0000000A0A20F424000000040ED9754C000000020A20C4240000000A0239A54C0FFFFFFFFA3094240FFFFFF9F699E54C00000006020FF4140FFFFFF9F71A054C0FFFFFF3F7CFC41400000002077A354C0000000C061FA4140000000005BA654C0000000604DFB4140FFFFFF7FC5A654C0FFFFFF3F7400424000000020E7A554C0000000A0DF02424000000080AEA654C0000000A080054240000000E031A854C0000000A0F4064240FFFFFF7F35A954C0000000E09E064240000000C0D8B054C0FFFFFF1FF1FF4140000000A082B154C00000008066FE414000000040B1B154C00000000071FA414000000040C1B354C000000020DFF5414000000020CDB654C00000004045F94140000000A057B954C0000000E063F941400000002080BA54C000000060C4F641400000006009BA54C0000000C0A4EF4140000000C09DBD54C0000000605BE54140000000202BBF54C00000002012E3414000000060D6C354C0000000802BE44140FFFFFF5F90C754C000000080C4E14140000000C031C954C0000000A09FE14140000000C0DCCB54C0000000204CDD4140000000409BCF54C000000080EEDB41400000002036D054C0000000C0B0D94140FFFFFF7F95D054C00000000073D841400000004019D354C00000008002D4414000000020F4D554C0000000009ED3414000000020C6D854C00000002007D04140000000A054DD54C0000000A073CC4140000000005FE054C0000000609FC7414000000000E8E354C00000000010C741400000008049E754C0000000C02EC941400000006010EB54C00000000051C8414000000000A8F154C0000000C0BBC641400000000022F554C0000000A070C241400000000066F854C0FFFFFFDF5AC14140FFFFFF9F3DFA54C000000040FDBC4140000000E019FD54C0000000804CBA4140 Tennessee 47 0.32812500000000000000 0.31400966183574879227 0.31441543700340522134 0.29289940828402366864 0.32587859424920127796 0.36029411764705882353 0.36565096952908587258 0.35023041474654377880 0.35194942044257112750 0.37735849056603773585 0.34593993325917686318 0.33106134371957156767 0.37457044673539518900 0.36007702182284980745 0.45699937225360954175 0.54033771106941838649 0.55474452554744525547 0.49630051223676721685 0.50395480225988700565 0.53138766519823788546 0.52686980609418282548 0.49542168674698795181 0.50746268656716417910 0.49144764288694201085 0.52145484266448712709 0.52753623188405797101 0.52512861100118717847 0.51727901054929065115 0.52526690391459074733 0.55364335408275357012 0.56426553672316384181 0.55297334244702665755 0.55555555555555555556 0.55621859296482412060 0.57076923076923076923 0.57777124375183769480 0.59391571308958972928 0.60196179659266907589 0.58951132300357568534 0.61692550078775602071 0.61213121518465029915 0.62652259332023575639 0.65113207547169811321 0.66842197647884851676 0.68867168722961063932 0.68927051225598121239 0.69305152645393010084 0.70691185795814838301 0.70110192837465564738 0.70501022494887525562 0.69437608239905204630 0.66878366428169466999 0.66325279480706815723 0.65054015969938938469 0.65042091271599468321 0.65147895335608646189 0.65272078025902041251 0.64851781614931629903 0.63525919159625485271 0.62106885491731578290 0.61569174710237624530 0.62915170556552962298 0.65156535010981647619 0.65793609219486642221 0.66688061617458279846 0.67784619415695008516 0.68238019219332018656 0.67070984703632887189 0.65524216204067872795 0.65619993591797500801 0.65073791348600508906 0.64564468503937007874 0.64001040705787743324 0.65793293829275838271 0.67222510495778102741 0.66062343086938167922 0.65424694823699544894 0.63195979113292785481 0.62262191730020396811 0.62798928990610328638 0.63546723300970873786 +20 0106000020E6100000050000000103000000010000000900000000000060D61F56C0000000204316484000000020042856C0000000C042044840000000A0B13956C0FFFFFF1FEDFA474000000080D34156C0000000C0E5EC474000000060F24856C0FFFFFFBF7EE9474000000040574C56C0FFFFFFBF1FEC4740000000A0E84C56C0000000C025F1474000000020FC4956C0FFFFFF3F3CF8474000000060D61F56C000000020431648400103000000010000000D00000000000040092056C00000002027A54740FFFFFFBF041C56C0000000A090AD4740000000C0850D56C00000008055B94740000000C06EF255C00000002046BC4740000000C012ED55C00000000041B547400000008031EF55C0000000C051B24740FFFFFFFFAEFA55C000000020D6AD4740FFFFFFFF370E56C0000000C0B4994740000000006A1A56C0000000407C7E4740000000601D1E56C000000020478E474000000060062656C00000006042914740000000E01C2656C000000080309F474000000040092056C00000002027A54740010300000001000000AC000000000000E0057755C00000002018FC4640000000C08C7A55C0000000A09FFA464000000000B07A55C0000000E087F5464000000040568455C0000000006CFB464000000060969055C0000000E035F9464000000020319455C000000040EEF34640FFFFFF7FFE9555C000000040CEEA464000000020529D55C000000060A2E1464000000040DFA155C0000000A0DEDF46400000006066A155C0000000C0AFDC464000000000DDA455C0000000A0E7DA4640000000404CA855C00000002085CF4640000000C0D5AB55C00000006035D346400000002098AC55C0000000E0A4D84640000000006AA555C0000000002EE84640000000A0B9B055C000000080BDE9464000000000B2B955C0000000C07EDB464000000080E9C755C0FFFFFF9F1FD9464000000060ADD055C0000000A004C746400000004041D555C00000008044B64640000000405CE555C000000000D594464000000000EAE555C0000000E0E38D4640000000A00DEB55C0FFFFFF1F02924640FFFFFF1FB1EE55C0000000A09B964640000000201CEF55C0000000E07B994640000000602DEE55C0000000C0189B4640000000400DEE55C0000000404F9E4640FFFFFF3F1FED55C000000040A19F46400000004014ED55C000000060D8A24640000000C04BE955C0000000A091AC46400000004030E955C00000002052AE46400000008020EC55C0FFFFFFFF15B24640000000A0A2F055C0000000A02CAD464000000040FCF455C000000060DEAD4640000000C0D5F555C0FFFFFF7F4FAC4640000000C02AF755C00000008062AF46400000004094F755C000000000A1AF464000000060EDF755C0000000C059AE4640000000408BF855C000000040D5AE4640000000E05CF655C0000000A0FCB34640000000C00CF755C000000000FAB84640FFFFFF5F10F455C000000000B6BB46400000006083F255C000000080E2BF46400000008085F355C000000020B4C54640000000E005F555C0000000C0C8C84640FFFFFFFF50F255C0FFFFFF5FC6C84640000000E098F155C0000000A0DACC4640000000C0A8F155C0000000807ECE4640000000E077F455C000000040C6D34640000000A048F455C0000000C02CD5464000000000F9F155C0000000A085D64640FFFFFF1FC0F155C0000000E091D746400000002044F355C000000040C6D94640000000A04AF355C00000002010DB464000000040E7F555C00000008079DC464000000080E7F755C00000004018E046400000000005FE55C0000000601CE24640000000405BFF55C000000080C5E54640000000004C0356C0000000C0A0E44640000000C0AB0556C0FFFFFF5F52E54640000000004F0856C0000000A0E3E8464000000040C90756C0FFFFFFBFDEEA4640000000C02D0456C000000000D5EF4640000000E01E0656C00000002028F2464000000080FF0556C000000040D8F54640000000E01E0756C00000002092F64640000000A09E0956C000000000DAF74640FFFFFF1F860B56C0000000600EFA464000000040C00D56C00000006056F9464000000040731056C000000000CAFB464000000020231356C0FFFFFF7F22FB4640FFFFFF5F8E1456C0000000C0BEFB4640000000E0AA1756C00000002050FF464000000020D11956C000000040E2FD4640FFFFFF5F111D56C0000000601A004740000000A0F41E56C0000000A0E5FF4640FFFFFFDF9C1F56C000000020AA014740000000A0FD2056C0FFFFFF3F6302474000000020162356C0000000E07902474000000080D02456C00000000027014740000000E03B2656C0000000E0FD01474000000040622756C0000000C040FF4640000000C02F2956C0000000C028FF464000000020582B56C00000008095024740000000C0052D56C0FFFFFF7F6D024740000000607B2E56C0000000C0CA03474000000000773156C000000060B602474000000020C03156C0000000202E044740000000C0CB3256C0000000E0A804474000000020793356C0000000C06F03474000000040343B56C0000000406D094740000000000D3F56C000000020DB0C4740FFFFFFFF604656C0FFFFFFFFA5124740FFFFFFBF327B56C0FFFFFFFFEB264740000000A0228756C0000000E0942B4740000000405C8756C000000020BF2E474000000060108956C0000000006D32474000000060518A56C0000000A0A1384740000000C0868D56C000000000D0404740000000C0869056C0FFFFFFBF2141474000000040419156C000000060E242474000000040339356C0000000A03643474000000080579356C000000040AD454740000000E0109456C0000000609B46474000000080A99856C00000004015454740000000001D9A56C000000000CA48474000000040328156C000000040DF564740000000A0B57856C0000000406E62474000000020A17256C00000000092694740000000A0BD5856C000000080D56C4740000000A0B94D56C00000004033764740000000E0004856C0000000E0927F4740000000E0A93F56C000000040A37F4740FFFFFFDF7D3B56C0000000A0F7834740000000009F3856C000000060648D4740000000C0472856C0000000E0E99C4740000000008D2756C0000000C0CA904740000000C0B52056C000000060A48D4740000000E0D22056C0000000402E844740FFFFFFFF391C56C0000000C0D27E4740FFFFFF9F881C56C000000000D5764740000000407D1E56C0FFFFFFDF776D474000000040931C56C0000000E054664740000000805F0B56C00000006015794740000000A0190C56C0FFFFFFFF5473474000000000570256C000000080BA74474000000040A2F955C0000000A0757447400000000079EA55C000000080206B474000000000C5D755C0FFFFFFFF074147400000000013C755C0FFFFFF5F324047400000008066C055C000000060A7444740FFFFFF3FC2B755C0FFFFFFBFE23847400000000099B055C0000000004C3E474000000000D6A855C0000000800E36474000000020959D55C000000040D3474740000000E0778955C0000000A027564740000000402E8655C0FFFFFFBFE0534740FFFFFF3FDF7655C000000020F0584740000000E03C6055C0FFFFFF1F4B56474000000000B84E55C0000000A0DD60474000000080193D55C0000000C0AD62474000000080B84155C0FFFFFF5FDF58474000000080354155C0FFFFFFBF45464740FFFFFFDF4C4355C000000020B6404740000000400F4155C000000080FB3C4740FFFFFF7F993B55C000000000713E4740000000006E3355C000000020D6384740000000004E2855C000000060D03D4740000000C0A52455C0FFFFFF3F36344740000000809E1A55C000000080853D4740000000E0F01355C0000000008C3E4740FFFFFF9F9F0B55C0FFFFFF7FD51F4740000000807A1155C000000080881A4740000000E0CE0F55C000000060F1154740FFFFFF5FA90755C0000000208A16474000000060E40155C0000000A08010474000000040F70355C000000000170C4740FFFFFFDF53FF54C0000000E05203474000000080B9F954C0000000C0C00047400000006003FA54C000000080E8FA4640FFFFFF9F3F0755C00000002040FD464000000060AF1655C0000000E0E4FF4640FFFFFF1F1A2055C0000000E039FD4640FFFFFF9F792755C000000040E404474000000020182C55C00000008098044740000000E0D32E55C00000008086ED464000000080773655C00000000000F24640000000A0F04355C0000000C02A034740FFFFFF9F335855C000000060CE0C4740000000C09A6055C0000000A00B0D474000000020F06955C00000006087FC4640000000E0057755C00000002018FC46400103000000010000001100000000000000B3F654C000000020CB0147400000004045F354C0FFFFFF9F83FE46400000002069F054C0000000207F034740000000201CEB54C000000040A1044740000000408AEB54C0000000E02F09474000000060E0EE54C0FFFFFF5FC30A4740000000C097E954C0000000404E0D474000000060BAE554C0000000E0530B4740000000002DE254C0000000A081014740000000E048DE54C00000004067FE4640000000E008E154C0000000207DF64640000000C01BE554C00000000070F54640000000204DE854C0000000C00EFA4640FFFFFF1F83F354C000000040E7F746400000006094F654C0FFFFFFBFABFF464000000060B2F854C0FFFFFF3F44FC464000000000B3F654C000000020CB01474001030000000100000076000000FFFFFF5F6DB555C0000000C0FAE14440FFFFFF3F86A755C0000000E025F4444000000080EC9F55C0000000202E104540000000C0F39755C0000000E0EB1F4540000000A03C9255C0000000C00D364540000000A0F08D55C0000000C02C634540000000A0859155C0000000E07D8F4540000000E0A39D55C000000000D2BC454000000060A3A255C000000020E3D44540000000A0A79C55C000000080E6E2454000000080DF9955C00000002021E2454000000000C79B55C0000000A006E44540FFFFFF5F709B55C0000000A0F9E84540000000E0679D55C0000000809FF9454000000040099C55C0FFFFFFFF08F94540000000602FA155C0000000E0DC064640000000C0B99855C0000000207317464000000040669155C000000000F52C4640FFFFFF7F3A8F55C0000000E0D9424640000000C08B9055C0FFFFFF9FB1594640FFFFFFDFEF8655C000000040025E4640000000004D8555C00000004093634640FFFFFF7F438655C000000000E16C4640FFFFFF9F4F8455C000000040FA72464000000040EC7255C000000080347E4640000000600C6755C0000000E02799464000000000306455C0FFFFFF9F1C97464000000080C96955C0000000A0AB7A464000000060D46855C0000000C0A363464000000020AA6155C000000040AF614640000000C0E15C55C000000020266E464000000080A05855C0000000605B81464000000080F85855C000000060159B464000000020E25755C00000004003A34640000000C08B5355C0FFFFFF1F02A9464000000080F04555C00000004063AF464000000000183F55C0FFFFFFFFC3AF4640000000E0FB3A55C00000004077B44640FFFFFF7F3B4555C00000006079BB464000000060B44755C000000040EEC8464000000040FD4455C0000000A0A9D0464000000040EF3E55C0000000A083D74640FFFFFFDF343E55C0000000206EDE464000000040582E55C0000000A0E0E3464000000080C61D55C000000000AAD34640FFFFFF5F921455C00000004032D54640FFFFFF9F270D55C000000020C1D0464000000060A70855C00000006021C9464000000000C70655C0000000A0D6BF4640000000A010FB54C0FFFFFFFFF1BE4640000000A019F254C0FFFFFF5F68B44640000000C096ED54C0000000E0C8B44640FFFFFF7FE9E554C000000000BCAC464000000000BCDF54C0FFFFFF3F2EAE4640000000E055DF54C0000000201AAA46400000000038D954C000000020EEA2464000000020EEDA54C0000000E0EAA04640000000A084D954C000000020589B4640000000E003D454C0000000209F8C46400000002072DC54C0FFFFFFDFC0864640000000A0C6DB54C0000000406C81464000000060C1DD54C0FFFFFF3FBA7F4640000000007BDB54C0FFFFFF7F90764640000000E076D454C000000040296E464000000060F9D154C000000080015A4640000000007CD454C000000020FA41464000000000D9D654C000000020E52A464000000000DEE154C0000000E070214640000000605EE454C0000000E0C5154640000000A04CE654C00000006005094640000000C01BED54C000000080A2FF454000000080E9F754C0FFFFFFBF3DFB4540000000E0C6FA54C0000000805FF54540000000800AFC54C0FFFFFF9F60D94540000000A0BFEC54C000000060C0CC4540FFFFFFBFE5E954C000000040BFCD454000000000FBE154C000000020EBDC454000000060A2DF54C0FFFFFF1FF6D9454000000040DADD54C0000000E073DF4540FFFFFF3F80D754C0FFFFFF7F16EC454000000040DED454C0FFFFFF7F60F84540000000602CBC54C000000000F40846400000002096B354C0000000804B0446400000000097AE54C0000000C07AFC45400000008096A754C000000040D8E44540000000A0C5A654C0FFFFFFFFE6D84540000000E03FA054C0000000800B964540FFFFFFFFDF9A54C0000000C0787C4540000000E0359E54C00000000007734540FFFFFFFF4A9E54C000000080A5614540000000602BA154C000000080275145400000008057A954C000000060DB504540FFFFFF1F95A854C000000040AD55454000000080B6AE54C00000006031574540000000E082B454C0000000A060514540000000405BB354C000000040734E4540000000A0D8B854C0000000C0733F4540000000E0FFB754C0000000C0A03A4540000000807CBB54C0000000E0762E4540FFFFFFFFE3C654C00000002076254540000000A069CC54C000000080CF0E4540FFFFFF3F2BCC54C0000000005804454000000080E5DE54C000000080CFDC4440000000A0E5F054C000000060C6DB4440FFFFFFBF98F754C000000000A4DB4440FFFFFF1FFE1655C0000000609FDA4440000000C09A1855C00000004082DA444000000040963255C0000000E045D9444000000020773255C00000008065E14440FFFFFFFFDD3455C0FFFFFF7F83E14440000000E05C4C55C000000000A4E14440000000E0055355C000000060BBE14440000000C0346A55C0000000209CE14440000000A0267355C0000000E0B9E14440000000005F8455C0FFFFFFBFDDE1444000000000038F55C0FFFFFF7FE5E14440000000409CA155C0FFFFFFBFFBE14440FFFFFF5F6DB555C0000000C0FAE14440 Michigan 26 0.68576388888888888889 0.63478260869565217391 0.61293984108967082860 0.58284023668639053254 0.55431309904153354633 0.66617647058823529412 0.73407202216066481994 0.71313364055299539171 0.72181243414120126449 0.71949685534591194969 0.69521690767519466073 0.66212268743914313535 0.71219931271477663230 0.67394094993581514763 0.84996861268047708726 0.86991869918699186992 0.80596107055961070560 0.75640295959021058623 0.82824858757062146893 0.86068281938325991189 0.84598337950138504155 0.82795180722891566265 0.85753052917232021710 0.82811848143512724239 0.89987740089906007356 0.85962732919254658385 0.88563514048278591215 0.82757366315023644962 0.82170818505338078292 0.82240937385573050165 0.83262711864406779661 0.83321941216678058783 0.79783037475345167653 0.81407035175879396985 0.83938461538461538462 0.86709791237871214349 0.89729277142059726486 0.89055240061951471347 0.84719904648390941597 0.87913571910871033086 0.85516814524448112234 0.82396856581532416503 0.84924528301886792453 0.87168685272950675794 0.88960102547668642846 0.86980772053427271393 0.86738499792789059262 0.89841471147748890298 0.91333792470156106520 0.90327198364008179959 0.88424026980220581533 0.83358790899589999196 0.80238009376126938334 0.76910689122995370060 0.77492246344705361099 0.77224118316268486917 0.78526888024303149816 0.77717337059586785108 0.73669787622744918931 0.71637439080268255092 0.70845447145016862426 0.71147516457211250748 0.71913040241223988386 0.70815435655666142832 0.72258631173569353422 0.74878815668806498100 0.75046170219425924187 0.73037165391969407266 0.71833913922912686819 0.71576951831677881021 0.71534351145038167939 0.72864173228346456693 0.71420799924312306346 0.72368577615953927798 0.74168592858153686495 0.70852496167433180031 0.69150610352600910212 0.65421547728472806211 0.63862414240682366030 0.64581499413145539906 0.65003033980582524272 +24 0106000020E610000001000000010300000001000000290100000000006060DE5BC0FFFFFFFFE259464000000080B8DE5BC0000000E082584640000000006FDD5BC0000000E0C5554640FFFFFF3F47DD5BC00000008089534640FFFFFF3F0ADE5BC000000060055246400000000071E05BC000000060A2514640000000A00FE05BC0000000C0DA4E464000000000E1E05BC000000080F04B4640000000C07EDF5BC00000000090464640FFFFFFFF91DD5BC00000002067464640000000A058DD5BC0FFFFFF3FDD44464000000080D5DE5BC000000000A3444640FFFFFF1F53DF5BC000000000AF434640FFFFFF7F40E45BC000000000C746464000000060AFE65BC00000006083454640000000C0C7EB5BC0000000C08146464000000040D6ED5BC000000000554446400000002008F15BC0000000A06B42464000000000ADF25BC0000000C05F424640FFFFFF7FA6F35BC00000004085404640FFFFFFDFC9F75BC000000080364746400000000022FC5BC000000040604646400000004087FE5BC000000040CF4346400000008075015CC0FFFFFF1F7F44464000000040AE015CC000000060EF424640FFFFFF3FBF035CC000000060AC4346400000004057065CC0000000405842464000000040E5075CC0000000A0A0434640000000A0B90C5CC0FFFFFF5F09444640FFFFFF3FE20D5CC0FFFFFF3FF0444640FFFFFF3FB10E5CC000000040A0474640000000C05F105CC000000000B0474640FFFFFF3F04125CC0000000605945464000000000DE155CC0FFFFFF5F3943464000000060BE155CC000000080A63F4640000000E078175CC0000000A08439464000000000E01A5CC00000002085394640FFFFFFBF4A1D5CC0000000C0053C46400000008010205CC000000080463B4640000000A076225CC0000000A0213D4640FFFFFF1FC0295CC0000000E08D3D4640000000C0A92D5CC0000000809E3F464000000060E72E5CC000000020013E4640FFFFFF7FDB315CC0FFFFFF5FAC3C4640000000A0E7325CC000000000A33A4640000000C0DA345CC000000000E93546400000000058345CC0000000608C3246400000008042345CC000000020A12E4640000000E0FA355CC000000000472D464000000020AD375CC0000000605E2F4640000000E0BB385CC0000000E04B32464000000000FF3B5CC0000000C02134464000000080003F5CC0000000A0C2374640FFFFFFFFB6405CC0000000E009384640000000405F405CC000000020F2394640000000E03E415CC0000000A0AD3D4640FFFFFF7F70405CC0FFFFFF7F5E414640000000C05D425CC0000000C03A444640000000E07B425CC0FFFFFF5F374746400000008042455CC000000000984A4640000000806B435CC0FFFFFF5FEB4F464000000060A0445CC0000000407A5646400000004047465CC000000060165946400000004074465CC0000000608D5B4640000000C019485CC000000000655E464000000060CB485CC0000000607961464000000080534F5CC0000000E0EC674640FFFFFFFF66505CC000000080C06746400000000057545CC0FFFFFFFFE0634640000000A0BE555CC0000000C0B86346400000008058565CC0000000E06067464000000080E95A5CC000000020B96A4640000000E0755C5CC0000000E0F76C464000000020B35F5CC0000000A02277464000000040225F5CC00000006046784640000000209A5D5CC0000000C06D78464000000020AA5C5CC0000000608C794640000000602B5C5CC000000040C77F4640000000604F5D5CC0000000E08583464000000060175D5CC0FFFFFFDF8E854640FFFFFF1F115F5CC00000008079874640FFFFFF3F505F5CC00000002020894640FFFFFF1F43615CC000000080838A46400000000099605CC000000000D08D4640000000E049635CC0000000A0C68D464000000000B4645CC0000000A0138F4640FFFFFF3F86645CC0000000403B91464000000020F7655CC0000000402D9346400000000067665CC0000000402D974640000000C042695CC0000000A07A9A4640000000401C6C5CC00000000095A1464000000020056C5CC00000002091A34640FFFFFF5F3E6F5CC0000000802AA94640000000E0626F5CC0FFFFFF9FF4B04640000000408B715CC00000002083B446400000008039725CC0FFFFFF1F0DB94640000000402B715CC00000002028BD4640FFFFFF9F5E715CC0FFFFFFBFE9C04640FFFFFFFFEB715CC0FFFFFF3F2BC24640000000C04C755CC0FFFFFF1FEBC14640000000E061735CC0FFFFFF3FBACA4640FFFFFFBF94745CC000000040E4CC4640000000C078765CC0000000C008CE4640FFFFFFDFC0795CC00000008089CE4640000000C0AE795CC0FFFFFF1F94D14640FFFFFF3F0C7B5CC0000000C0DDD34640FFFFFF1F407B5CC0000000E0ECD54640000000A0A57D5CC000000080F8D6464000000060187E5CC00000004046D94640FFFFFF3F8C805CC0FFFFFF5FDCD74640000000C036815CC0FFFFFF1F13D6464000000020A5805CC00000002088D346400000008018815CC00000008007D24640000000008F835CC00000006007D046400000006040855CC00000002011CB4640000000A080875CC0000000601DC946400000008064885CC0000000A075C6464000000000FE8A5CC000000000A2C54640000000C0688C5CC00000008095C34640000000E06D8F5CC0FFFFFF1F87C4464000000040CE8F5CC00000004063C04640FFFFFF7FB9905CC00000006033BE464000000040D5945CC0000000A08FBA4640FFFFFF5F5B965CC0000000E052BB4640000000E0B6975CC0000000802FBE4640000000A0C29A5CC000000040E2BF464000000040B09B5CC0000000408CC34640000000E08D9D5CC0000000A022C6464000000000B99F5CC000000060FBC54640000000A0B1A15CC0000000A075C74640FFFFFFFFD6A35CC0000000E03FC64640FFFFFFDF8FA25CC00000008059CC46400000008014A45CC0000000E0EACF4640000000600DA05CC00000006084D3464000000040A0A05CC0000000404AD64640000000E0C89F5CC0000000E0E4D84640FFFFFFDF2DA25CC0000000E08DDC464000000040A0A25CC0000000A07FDF4640000000E001A45CC00000000099E14640FFFFFF7F0DA15CC000000000B3E7464000000020E39F5CC0000000C0DFEB4640FFFFFF9F439E5CC00000006076EB4640000000C04E9C5CC00000006025ED4640FFFFFFBF059A5CC0000000405BEC46400000002015995CC0000000E07BEF4640FFFFFF1F689A5CC0000000E092F44640000000206D9B5CC0000000A0F5F54640FFFFFF1FE1995CC0000000A01EFA464000000060569A5CC0000000406CFC4640FFFFFF1FF29E5CC0FFFFFF9FB4FE4640000000804F9E5CC0000000A042014740000000C0939F5CC0000000C001034740FFFFFFBFBF9D5CC0000000C083064740FFFFFF5F209D5CC000000020890A4740000000007E9E5CC000000080C00D4740000000605CA05CC0FFFFFF5FE00E47400000002027A15CC0000000206D114740000000408EA05CC0000000E02814474000000060D49D5CC000000040E213474000000020289C5CC000000060A3154740FFFFFF5F129C5CC0FFFFFFDF331C4740000000A0339E5CC0FFFFFFDF361F474000000060439E5CC0000000A06320474000000040939B5CC00000000074244740000000C02A9A5CC0000000804D324740FFFFFF5F59995CC0000000E026334740FFFFFF7F84985CC000000020D1364740000000C0399A5CC0000000C0643E47400000008002975CC000000040CB404740FFFFFFDF58965CC0000000603C424740FFFFFF7FE9955CC0FFFFFF3F424B4740000000A0B8945CC0000000C0BB4F4740000000005C955CC000000040C05347400000004084985CC0000000C0B1544740000000A0329C5CC000000060A952474000000020F69E5CC0000000E0D34F47400000004086A25CC000000000A95147400100004008A75CC00000004087504740000000E033A95CC00000008098544740000000C038A95CC000000060E3554740FFFFFF9FFFA75CC0000000C0F55747400000006011AB5CC0FFFFFFFF0D5E474000000080A3AC5CC000000080EE5D474000000060D1AF5CC0000000C0FC584740000000C00BB25CC0000000E0FF594740FFFFFF5FB8B15CC000000000BE604740010000C0C1B25CC00000006020624740FFFFFFDFBFB55CC0FFFFFF7F47634740FFFFFFBF66B75CC0000000400866474000000000B0B95CC00000008056664740FFFFFFFFA2BC5CC0000000A01F6D47400100008022BC5CC0000000400B7247400000002015BB5CC0000000A02074474001000060AEBD5CC00000000070764740000000000AC05CC0FFFFFFDFBC7A4740FFFFFFDF53C25CC000000040467B4740000000C07FC35CC0000000A0997C4740010000C024C55CC0FFFFFFBF67834740000000409CC85CC000000000258847400000002074C95CC0000000C0C08B4740FFFFFF3FFACA5CC0000000C07F8C4740000000604BCC5CC0000000A0E28F474000000080E5D25CC000000000FE964740FFFFFF7FC0D45CC000000080639F474000000080EED55CC000000000A7A04740FFFFFFFF0FDA5CC0000000E0BFA14740000000003EDB5CC00000002021A34740FFFFFF7F0FE05CC0FFFFFF5F0FA44740FFFFFFBF69E15CC0FFFFFF1F62A54740000000807DE35CC000000000D7AA4740000000A04EE65CC0000000A05EAF4740000000A0D1E85CC0FFFFFF9FA7B04740FFFFFF5F97EA5CC0000000401AB3474000000020F5EF5CC0FFFFFFFF15B6474001000080F1EF5CC0FFFFFF7F8EB74740000000C0D0EE5CC00000000002B9474000000020EDE95CC0FFFFFFFF80B94740000000C019E95CC0000000409BBA4740FFFFFFDFE7E85CC0000000C0D6BC4740FFFFFF1F46EC5CC000000080ABBE4740FFFFFF7FD5EC5CC0000000E0AEC24740000000407AEF5CC00000004052C44740FFFFFFDF3AEC5CC0000000009FCB4740FFFFFF5FA0EC5CC000000000DECE4740000000A0EAEE5CC0000000E0E9D1474000000000E4EE5CC0000000400AD947400000002095F15CC0000000E0DADA4740FFFFFFBF87F25CC00000004059DF4740000000C07CF55CC000000040D0E04740000000E04AF65CC0000000A012E74740FFFFFF7F9AF75CC0FFFFFF5FECE9474000000060C9F95CC0000000C0AAEB474000000020F5FB5CC00000006000EF474000000000DEFF5CC0000000806DF64740000000408E015DC0000000A085FB4740FFFFFFDF5B035DC000000060F6FC4740000000E07C035DC0000000C0B11A48400000004090035DC0FFFFFF3FD63F4840FFFFFFFFFF035DC0000000E0FF7F4840000000009DAE5CC000000000008048400000000000845CC00000000000804840FFFFFFFFFF0B5CC0000000E0FF7F4840FFFFFF3F0AD25BC00000000000804840FFFFFFFFFFAF5BC000000000008048400000000000605BC0000000E0FF7F48400000000000105BC0000000E0FF7F48400000000000CC5AC0000000E0FF7F48400000000000885AC0000000E0FF7F4840FFFFFFFFFF435AC0000000E0FF7F48400000000000045AC000000000008048400000002051035AC000000080A9524840000000C04D035AC0000000200C324840FFFFFF5F11035AC0FFFFFFFF0100484000000020FF025AC00000004033B34740FFFFFF7FE8025AC000000080BAAA47400000004001035AC0000000C04B52474000000040F5025AC0000000A071454740000000C007035AC0000000C0F32347400000004019035AC000000000B4F846400000004023035AC0000000E007F1464000000060C6025AC0000000E03F9B4640FFFFFF9FB9025AC0FFFFFF7FB87F464000000060CC035AC000000020A97F464000000000A4425AC0000000C023804640000000E067455AC000000080FA7F46400000002051815AC0FFFFFF1FA57F4640000000A08D905AC0000000A0827F4640FFFFFF9F32F95AC0000000E0F97F4640000000608C105BC0000000400580464000000000F9275BC0000000C0B27F4640FFFFFF1F1D735BC000000040F27F4640000000A0AA7F5BC0FFFFFF7F5D804640000000A016995BC000000000D57F464000000000739B5BC000000040057F4640000000A05EC35BC000000040757F46400000000041C35BC000000060105546400000002040C35BC0FFFFFF3F983C4640000000C001C65BC0000000C03B3E46400000008033C85BC0FFFFFF3F1B404640FFFFFF9F8CC85BC0000000C09443464000000080D8CA5BC000000020CB4546400000002064CB5BC0000000804F484640000000A0FFCD5BC0FFFFFF1F60494640000000E0F0CE5BC0FFFFFF3F234D46400000006004CE5BC0000000801C4F4640000000C048CE5BC00000000041504640000000C045D15BC00000008036524640000000403ED15BC0000000A04156464000000060DFD25BC0FFFFFFFF6C574640FFFFFFDF23D45BC000000040465A46400000004061D45BC0000000202D5D46400000002059D65BC0FFFFFFDFF55C464000000000C7D75BC000000080615F46400000004096D85BC0000000406F5E4640000000203CD95BC0000000C0BE5A46400000008057DC5BC0FFFFFFFF4B5B46400000006060DE5BC0FFFFFFFFE2594640 Montana 30 0.51388888888888888889 0.48405797101449275362 0.43359818388195232690 0.50147928994082840237 0.47603833865814696486 0.53529411764705882353 0.65927977839335180055 0.54723502304147465438 0.53951527924130663857 0.65031446540880503145 0.59288097886540600667 0.55404089581304771178 0.61254295532646048110 0.57830551989730423620 0.72190834902699309479 0.73983739837398373984 0.73357664233576642336 0.74445076835515082527 0.83841807909604519774 0.90418502202643171806 0.78227146814404432133 0.79710843373493975904 0.81682496607869742198 0.76053400083437630371 0.73968124233755619125 0.73333333333333333333 0.74950534230312623664 0.70170971262277191706 0.70569395017793594306 0.75723178322958623215 0.71433615819209039548 0.70915926179084073821 0.66568047337278106509 0.74214824120603015075 0.71692307692307692308 0.69567774184063510732 0.71113591962042980742 0.70469798657718120805 0.66865315852205005959 0.66509115462525320729 0.67753249432638745616 0.71218074656188605108 0.71490566037735849057 0.76443742320519571704 0.80307643005928537093 0.78966681344488477910 0.80038679375604365244 0.78630310716550412175 0.76170798898071625344 0.78946830265848670757 0.75644881961534955793 0.73502693142535573599 0.73883880274071402813 0.71609743004764141448 0.69909487942274827521 0.66581342434584755404 0.63422693599104620796 0.62206807066573510330 0.59356017355560630281 0.55658766193193651852 0.56684885839438694422 0.58064033512866546978 0.61456278152105126010 0.59766020604155753449 0.61049929058847375177 0.59377047032621511856 0.58734779478511284315 0.57908102294455066922 0.56655242162040678728 0.56936879205382890099 0.56027989821882951654 0.55533956692913385827 0.57574682466472716952 0.58777753980152780752 0.61236379074484645502 0.61136661556577573374 0.61645612691931436349 0.60512780396221618133 0.60495086222881513073 0.61056704812206572770 0.62005081917475728155 +9 0106000020E61000000200000001030000000100000055010000FFFFFFBF554855C00000004058C73F40000000A06C4855C0FFFFFF5F9DC83F40FFFFFF3FAE4855C000000020CFDA3F40000000C05F4755C00000008042E83F40FFFFFF1F7D4455C0FFFFFFFF04FB3F4000000020A24355C0FFFFFF3F3402404000000020F94355C0000000A06106404000000060A14355C000000000E5084040000000E0F54255C0000000609D0B4040000000A06A4355C0000000A030104040000000A0904155C0000000804315404000000060814055C000000080E016404000000020783D55C0000000208C184040000000C0733E55C000000000261B4040000000A06B3B55C0000000C0E11B4040FFFFFFFFA03A55C0000000603C1D4040000000C01E3B55C000000080A51F4040000000A0F23955C000000080EC1F4040000000C0383955C0000000C02721404000000060423955C0000000006122404000000000F43A55C0000000A08025404000000080584055C0000000402B2A404000000060234055C000000040682C4040000000E0303E55C0000000A0882F404000000000033F55C00000008087314040FFFFFF5F253E55C000000000C632404000000000CA3D55C000000080F3364040000000405A3F55C000000060333A4040000000E0B93F55C0000000A067424040000000A0884455C000000000624A4040000000C0634555C0000000C02A4D404000000060874555C0000000A06C504040000000C0A54655C0000000C0A852404000000060CB4555C0000000C08756404000000020E64655C0000000A04C584040000000A04A4755C000000020FC5D404000000020FF4755C0000000A0435F404000000000894855C0000000A0CA604040000000A0334855C0000000C07263404000000020644A55C00000002056674040000000E0914B55C000000060936F4040FFFFFF9FF24E55C0000000C05C8F4040FFFFFF3FF84E55C0FFFFFFDF86904040000000E0CE5255C0000000607FB64040000000A0855355C000000020C8BE404000000020765555C000000000D1D3404000000060945855C000000020E1F3404000000080545955C000000060D8FA4040000000C0A95A55C0FFFFFFFF1C0B414000000000765D55C00000000021254140000000E0C26055C000000060EF424140000000E0A76155C000000040D44A4140FFFFFFFF386255C000000000AB4F414000000060516555C0000000005E6E4140000000C0F96655C000000020BB7E414000000040EA5D55C0000000E0B97E4140000000000A5755C000000020B57E414000000000235155C0000000009A7E414000000020433E55C0000000000C7F4140FFFFFF3FF23D55C0000000C00D7F4140000000003B3355C0000000A0127F4140000000405B3155C0000000A0CE7E4140000000A0992755C0FFFFFF1F8D7E414000000000BA1455C0000000E0977E414000000040910755C0000000207E7E41400000006044FF54C0000000C0997E41400000002008FC54C000000060A47E4140000000A029E354C000000020A97E414000000020D5E054C0000000A0FA7E414000000060CDC654C00000002009804140000000E06FC654C000000020F67D4140FFFFFF1F67C754C000000040307A414000000020C7C754C0000000A0FB7A4140FFFFFFFF27C854C000000000267A4140000000203FC754C000000000D0774140FFFFFF7F0CC854C0000000E068784140000000E0CFC854C0000000E01677414000000000EEC954C0FFFFFFFF517741400000008024CA54C0000000A072754140000000E0AACB54C0000000208D7441400000006008CD54C00000008029714140000000200DCE54C0FFFFFFBFCD71414000000040E8CE54C000000060D66F41400000008090CF54C000000000627041400000008017CF54C0000000A05F6E41400000000007D054C000000040CF6C4140000000E008D054C0000000A0786B41400000002025D154C0000000E0686B4140000000804BD154C0FFFFFF7F516841400000004062D354C0000000A018674140000000E0A8D454C000000060C1644140000000E0B4D454C00000002046604140000000C06DD654C0000000C0175D41400000008073D654C0000000A0D25A414000000000C1D554C000000080BB5641400000000023D354C000000040D8544140000000C02CCF54C0000000207E4E4140000000608FCA54C0FFFFFFBFA64C4140000000402ECA54C000000060D2494140FFFFFF9FCCC854C0FFFFFF9FAC484140FFFFFF7FEDC654C00000002078444140000000C010C554C0FFFFFF5F6A424140FFFFFF1F87C354C000000000B73E41400000006084C054C0000000C0403C4140FFFFFF9FDABE54C000000080FA3C4140000000A0EEB954C0000000402B3D41400000004090B754C0000000608D3A4140000000E007B654C000000020CF344140000000808AB554C0000000007B2F414000000020C2B354C0000000607E2B4140000000E01CB254C0000000202C25414000000080EAB054C000000000F3234140000000E085B054C0000000A0DB1D41400000000089AF54C0000000004C1A41400000006019AF54C000000040B8154140000000A045AA54C0000000E0DA0D41400000000099A654C0000000206B0441400000008029A654C0000000E0B301414000000060B8A454C00000002001FC4040FFFFFF9FE9A454C000000000C6FA4040000000C021A154C00000000028F74040FFFFFFDF3B9D54C0000000A065F04040000000E02D9B54C0000000C018EE404000000000029954C00000006050ED4040000000406A9754C000000040FEEA4040000000607E9654C000000000EAEA4040000000E0EB9354C000000080E2E6404000000000909354C0FFFFFF9F29E44040000000400C9154C0000000C077E14040000000200B8F54C0000000C058D8404000000020B88D54C0000000001ED74040000000604F8C54C0000000A0D5CF4040000000E0B48A54C000000000BBCE4040000000A0E78854C0000000C000CC4040000000C0758754C0000000401DCC4040000000C0368454C0FFFFFFFF6FC94040000000207A8254C00000004010C64040000000A04C8154C0000000C0F0C44040000000E0C97F54C0000000009BC2404000000000CA7E54C0FFFFFFFFCBBE404000000060F07B54C0000000E045BC4040000000E0AC7A54C0000000C0C0B94040000000205B7B54C000000000CFB74040000000807A7A54C0000000A028B5404000000020317C54C0000000203AB4404000000020437B54C0000000802EB04040000000A07E7C54C0000000C041B04040000000C0F57B54C000000080D6AC404000000040577A54C0000000A0B7AC404000000060627A54C0000000C080AA404000000080317954C000000060E4AA404000000060217854C0000000C041A74040000000C0677754C00000002063A8404000000000C77554C0FFFFFF1F73A74040000000A0187754C0FFFFFF5F02A6404000000040E97454C0000000C077A24040000000A0C27554C000000000F7A24040000000C0907554C00000000057A1404000000000AA7654C000000040269F4040FFFFFF3FE67354C000000040F79C4040000000804F7354C000000080989A404000000080EB7154C0000000E0CD9B404000000060487154C0000000A05B9B4040000000E09F7054C000000060FD98404000000060DE7054C000000040B895404000000000656F54C00000008080924040000000A0156D54C000000080B78F4040000000C01C6754C0000000803A8B4040000000C0246654C0000000A00A89404000000060DA6354C0000000C0C3874040000000A00F6354C00000006092854040000000C0E76154C0000000609A854040FFFFFF1F926054C0000000209E814040000000C08B5F54C0000000A09880404000000060E05F54C000000080D17A404000000000A46054C000000000467A404000000080AA6054C00000006039794040000000C08A5E54C0FFFFFF5FDE724040000000E0C55D54C000000080E3724040FFFFFF9FEF5E54C0000000801E704040000000805B5D54C0000000C0846F4040FFFFFFBF265D54C0000000C0156C404000000020AE5B54C0000000E0B66B404000000000255B54C000000020726A4040000000008D5B54C000000060FA684040000000C0F15A54C00000002098674040000000E08B5B54C0000000409B644040FFFFFFBFA35A54C0FFFFFFFFD6604040000000E0155B54C000000060F45F404000000060195A54C000000000EE5E4040000000E0EC5A54C000000060C759404000000040125A54C000000040E0574040000000E0935954C00000004048534040000000C0625A54C0000000400E50404000000020A55854C000000020404C4040000000C0735754C0000000C0494B4040000000E07A5754C000000080784A4040000000A0815654C0FFFFFF3FAB4A404000000000CC5554C0000000801C49404000000060255354C0000000009748404000000000915154C000000040FE46404000000080295154C0000000E05144404000000060214F54C0000000809D42404000000040804C54C0000000A0703B404000000060CE4C54C0000000C0C6354040000000406B4B54C0000000607D31404000000080794B54C000000040902F4040000000E0164A54C000000080552B404000000060174954C000000040942C4040000000A0734854C0000000E08C2A404000000020F34754C0000000A062234040FFFFFF5F7E4954C000000040F620404000000080764954C000000020B41C4040000000405A4754C0000000C05E18404000000040A24754C000000040070F404000000000B94654C000000080780D404000000060463954C000000060BD00404000000080513E54C0FFFFFF3F9DF23F40000000A0253E54C000000040DDE33F4000000020F64854C00000004037DD3F40000000E0874B54C0FFFFFF7FDAE73F4000000080464D54C00000006060E63F4000000060A34C54C00000006070EB3F40000000A00F4D54C0FFFFFFDF9FED3F40000000C0594F54C00000006038E73F40000000A0345254C0FFFFFFFFFFF23F4000000000B04F54C0000000A0FCE43F40000000E0394D54C0000000E0A1EB3F4000000000194E54C0000000E096E43F40FFFFFFFF8B4B54C0000000E0BFE53F40000000A0104954C0FFFFFFDF71DA3F4000000000824254C0FFFFFF3FB9D23F4000000000EB4354C000000080FEC63F40000000A0C44854C00000000013BA3F40000000E0254B54C0FFFFFF1FB2CC3F4000000020944C54C000000040DDC83F4000000000394B54C0FFFFFF1F4FBC3F40000000C08D5254C0FFFFFFBFA8CC3F40000000C0A54854C00100004056A53F40000000E0FD4B54C00000006084993F40000000E06A4F54C0FFFFFFFFD3A33F4000000020514F54C000000060818E3F40000000207A4C54C00000000043813F40FFFFFF3F5C4D54C0FFFFFFDF77773F40000000400C5454C0000000E05D563F40000000A0685754C0FFFFFFDF25583F4000000000465A54C0FFFFFF3FAE4F3F40000000E0465954C0000000608C433F4000000040E55854C0FFFFFFFF024C3F40000000E0435354C0FFFFFFFF8C463F4000000020DE5354C000000020123E3F4000000060655854C00000002012263F4000000000D46154C0FFFFFF7F82213F40000000C0A06154C0010000E018163F40000000E06E5E54C0FFFFFFBF280B3F4000000060476254C00000002097133F40FFFFFF1F6D5F54C0FFFFFFDF4EFC3E40000000E0D66154C0FFFFFF1F3EF63E40000000600A5F54C0000000A0D5F13E4000000080E26154C0000000C052DD3E4000000080E85F54C0FFFFFFFFDCC13E40FFFFFF7FD76154C0000000A0A1B83E40000000805A6254C0000000C0D2B43E40000000807C6654C0000000C082B93E4000000080B76654C00000006051B73E40000000C0336854C0FFFFFF1F46BB3E4000000040216954C0010000A095BA3E40FFFFFF5FE26D54C0FFFFFF1FC1BE3E40FFFFFF7F286F54C0000000E07EC33E4000000020817054C000000040F7C43E4000000060B87154C0FFFFFFDFE7C23E40000000606A7354C0000000A0B0C93E4000000000E27754C00000008081CC3E40000000408E7954C0000000C0FCD33E4000000080177A54C0000000A030D03E40000000006A7C54C0000000C0F2D23E4000000020017D54C000000080F8D13E40000000C08C7D54C0FFFFFFDFB7CB3E40FFFFFF3FD37E54C0FFFFFFBF32C73E4000000040018154C0FFFFFF9F84CA3E4000000020D78054C0FFFFFFFF96C33E40000000000D8254C000000060DEC13E40FFFFFF7FE28254C0000000C092A63E4000000060E28054C0FFFFFF5F34993E40000000C0618054C0000000001D923E4000000080738154C0FFFFFF3F3C7A3E40FFFFFF1F428254C00000000051713E4000000000F68254C0000000A00F6F3E4000000020778254C000000060F0603E4000000040638354C000000020125D3E40000000208C8A54C0000000406E5C3E40000000A08B8B54C0000000404F5E3E40000000A02A8D54C0FFFFFF5F726C3E4000000020C28C54C001000060627D3E40000000C00C8E54C000000080B3803E40FFFFFF7F448F54C0FFFFFF9FFD873E4000000020288E54C0000000C01C913E4000000040629A54C0FFFFFFBFBB933E4000000080A69D54C0FFFFFF9F1C953E40FFFFFF9F51A554C0000000E0BD963E400000008093AC54C0000000E072983E40000000207CC854C000000060029F3E40000000E04AD354C0FFFFFF5F11A23E4000000000E3D354C00100002054A23E40000000C0FCE654C0000000A08CA63E40FFFFFFBF3AEF54C0000000E008A93E40000000C00D0055C000000020E3AC3E40FFFFFFBFD70455C0FFFFFF5FAAAD3E4000000060081255C000000080B2B03E4000000080F81755C0FFFFFF3FA3B13E40000000C03C3755C0FFFFFF9F64B63E40000000A0563755C0FFFFFF1FDDB63E4000000000DB3855C0FFFFFFFF62BE3E4000000040963A55C0FFFFFF7FFFC03E4000000080533B55C001000040A1C63E4000000060B03B55C0FFFFFF5F6CCD3E4000000000543B55C0FFFFFF7FC3D83E40FFFFFFBF053C55C0000000A00FE53E4000000080053E55C0FFFFFFBF49ED3E4000000020503E55C000000080A6F63E4000000080034055C000000080A8FA3E40000000801B4055C00000000046003F40000000A0174155C0FFFFFF9F75143F40000000E0704255C00100006064203F4000000020664455C0000000E084293F40000000E0FA4555C0000000400A2C3F4000000000984655C0000000C05D323F40000000C0FE4555C0000000E0153A3F40000000A09E4655C00000004071453F4000000000344555C0000000208B4D3F40000000E0494555C0FFFFFF5F08553F4000000000A24555C0FFFFFF9FF75D3F4000000040EF4355C0FFFFFFFFC3703F40000000A03F4455C0010000C0D5793F40000000C0BC4255C001000040FD843F4000000020BD4255C000000060E18D3F40FFFFFF7FCC4355C000000040009F3F40000000201F4755C0000000A01FAF3F40FFFFFF7F994755C0FFFFFF1F5AB53F4000000000624755C0000000203DBB3F40FFFFFFBF554855C00000004058C73F4001030000000100000008000000000000200E5F54C00000008055E73E40FFFFFF5FA95A54C0FFFFFFDF69F83E40000000A0CE5954C0FFFFFFFF45F03E40000000802D5D54C0FFFFFF3F58B73E4000000080B25E54C0FFFFFFDF94BC3E40000000A0155E54C0000000802ADC3E4000000020396054C000000000AAE13E40000000200E5F54C00000008055E73E40 Georgia 13 0.30121527777777777778 0.29661835748792270531 0.29057888762769580023 0.29585798816568047337 0.32587859424920127796 0.35882352941176470588 0.37119113573407202216 0.34792626728110599078 0.32982086406743940991 0.36477987421383647799 0.34371523915461624027 0.32814021421616358325 0.35996563573883161512 0.36392811296534017972 0.45511613308223477715 0.51969981238273921201 0.53467153284671532847 0.48093340922026180990 0.50112994350282485876 0.54405286343612334802 0.53684210526315789474 0.51325301204819277108 0.54454997738579828132 0.53400083437630371297 0.54188802615447486718 0.53913043478260869565 0.56311832212109220419 0.54528919607129865406 0.54199288256227758007 0.57964115708531673380 0.58615819209039548023 0.58031442241968557758 0.57330703484549638396 0.57914572864321608040 0.60338461538461538462 0.61217289032637459571 0.63019815796818308680 0.63551884357253484770 0.63289630512514898689 0.65068647310375872158 0.65236228595007220961 0.66679764243614931238 0.69169811320754716981 0.70879410215903106898 0.72055760294824547348 0.71436958755320710407 0.71170051112032048626 0.72213062777425491439 0.71407254361799816345 0.71462167689161554192 0.70385561936013125513 0.68124447302837848702 0.68049044356292823657 0.67462926927464268939 0.68668903095132603329 0.69311717861205915813 0.70047433779246389170 0.69817347040622816648 0.67686686458095455583 0.66130711875702919982 0.65135480869868589371 0.66345002992220227409 0.68082492647879983621 0.67515278505325650428 0.67998783865955003040 0.69337088955849600419 0.69583998497511503428 0.70464866156787762906 0.68959995505112934038 0.69779985047527501869 0.69567430025445292621 0.68750000000000000000 0.67636415241609309586 0.68203993241474500845 0.68540968913628001321 0.66642227110133528850 0.66985666283867126026 0.64027144896642090235 0.62849990728722417949 0.62586194248826291080 0.62738925970873786408 +21 0106000020E61000000100000001030000000100000032010000000000E0BBEE56C000000000F3BF4540000000A0F30457C0FFFFFF5FE5BF4540FFFFFFDFFD1C57C000000080EFBF454000000080B32357C0000000A009C04540000000A0BA4157C0000000202BC04540000000C0774357C00000000031C04540000000200A6057C00000008011C04540000000A0D26957C0000000801AC0454000000060517E57C0000000400BC0454000000060C78F57C0FFFFFFFFDEBF4540000000801E9D57C000000040C3BF4540FFFFFF3F03B757C00000006002C0454000000080E4BA57C0FFFFFFBFECBF4540000000A05CD957C0FFFFFF1F0CC0454000000040BADD57C000000020F2BF4540000000A076F757C000000080DEBF454000000020E30358C000000000D1BF4540000000E0721D58C0000000A0F7BF4540000000C0331D58C000000080A4EC4540000000C0401D58C0000000E07B194640000000201B1D58C0000000A0E944464000000080351D58C0FFFFFFFF7D504640000000E01C1D58C0000000C093664640000000002B1D58C0000000806B7C464000000000111D58C0000000C03AA34640FFFFFFDF431D58C0000000E041A64640FFFFFF5F8B1E58C0000000C00DAA4640FFFFFFBF0F2258C00000008005B04640FFFFFF1FB42658C000000080C2B2464000000040572C58C00000000091B4464000000040362F58C000000060A7BA4640FFFFFF9F353158C0000000003EC2464000000060EF3558C0000000C0C4CA464000000060B23658C000000000F9CD4640000000C0463558C0FFFFFFFF4AD34640000000000D2A58C0FFFFFFDF97DE464000000040AC2658C00000008076E74640000000609B2558C0000000C0B0E84640FFFFFFBF422458C00000004092F74640FFFFFFDFEE2358C0FFFFFFFF4EF9464000000000DB2458C0000000A0BA024740000000204D2358C0000000A03B0C474000000040882458C000000000AF164740000000409A2558C00000002092184740000000C0822558C000000000941B4740000000E0422658C0000000808E1E4740000000A0822658C0000000000E2B474000000020542758C0000000C0E82C474000000040B62958C000000040032E474000000020062C58C0000000E0C4344740000000A0652D58C0000000C0AE36474000000000BB2D58C000000040003C4740000000805B2F58C000000000A73E4740000000400E3058C000000020594B4740FFFFFFDF523158C000000080CD4C4740FFFFFF5F2C3258C000000020E44F4740000000808D3258C0FFFFFF9F9D50474000000000C63258C000000040E456474000000020FF3158C0000000A0815A4740FFFFFFDFEB3158C0000000A09461474000000060FF3258C0FFFFFFDFF167474000000020253158C0000000A0256C4740000000E0C53158C000000080066F4740000000606B3058C0FFFFFFDF1E76474000000040D03058C0000000A0D877474000000080673258C00000000053774740000000C0843258C0FFFFFFDF5F794740000000404C3358C0000000405A7A4740000000A0C13258C0000000601E7C474000000020403458C0000000E0227C4740FFFFFF3FBF3458C0000000C071804740000000A06F3558C00000006050814740000000C09F3458C00000000059844740FFFFFF1FE73458C000000040188A4740FFFFFF1F673458C000000080DB8B4740FFFFFFFFAE3558C00000000072934740FFFFFF5FDF3458C0000000A0C5954740000000A08C3558C000000040C89E4740000000605A3658C000000020E1A0474000000040973558C0000000E09EA54740000000402B3658C0FFFFFFBF45A8474000000080A33558C000000080CFAB4740FFFFFFDF6A3658C0000000A034AE4740000000E0B93558C0000000A02BB14740000000A0603658C0000000E058B44740000000207B3758C0000000E0E0B4474000000000C03658C0FFFFFF5FE8B74740FFFFFFDF713758C00000008014BB4740000000007B3658C00000002015C04740000000A00F3758C0000000A0BCC24740FFFFFF3F533658C000000040B5C5474000000080EE3658C00000008010C84740000000E0843658C000000060F3CC4740000000E0DE3758C000000080C1CE4740FFFFFF7FE63858C0FFFFFFFF43D64740FFFFFF5F173B58C00000004068DB474000000040A03B58C000000060BBE1474000000040473D58C000000000B1E5474000000040F23E58C0000000C0A3E7474000000020853E58C0000000A0FDE94740000000C0FF3F58C00000006063EF4740000000204B4158C00000006013F0474000000060F54058C0000000207EF54740000000800D4358C0000000A03BFA474000000020454458C0FFFFFFDF2A06484000000060E94558C0000000A001094840000000C0504658C000000040ED0C484000000000C74758C0000000C0E50E484000000060B74758C0FFFFFFFF46124840000000E0B64858C000000040FF124840000000E0674758C0000000006214484000000020C64858C0000000007A15484000000040B34858C0000000606E16484000000060C34858C000000060F8184840000000C0594858C0000000C0141A484000000040134758C0FFFFFF5F931A484000000000EC4858C0FFFFFFFF621C4840FFFFFFFFF74658C000000040311D484000000020244858C0FFFFFF9FE41D484000000060E64758C0000000202D21484000000040B94858C000000020DB214840000000A0204758C0000000009223484000000080304758C000000000A1244840FFFFFF9F544858C0FFFFFFDF8225484000000080414758C000000040BF25484000000060524758C000000080DD26484000000060774858C000000060CE274840FFFFFFFF2E4758C0000000E0F328484000000020C14858C0000000A0BA294840000000A05E4858C0000000E0452E4840FFFFFF5F9A4958C0000000607E2E4840FFFFFF3F8A4858C000000020AD2F4840FFFFFF7FA14858C0000000E03431484000000060244A58C000000040B1314840000000E03D4858C0FFFFFF1F3634484000000000914958C0000000207B344840000000E0AE4958C0FFFFFF5FB635484000000000D34758C0000000A04335484000000060A24758C000000080F3374840000000402B4958C0000000801438484000000080974858C0000000E03742484000000080784958C0000000C04C42484000000000E64858C0000000E06F444840FFFFFF9FEE4958C0000000C0EA444840000000E03E4A58C0FFFFFF9FC5454840000000805C4958C0000000C057464840000000E0B94A58C0000000C0F8474840000000C0B64958C0FFFFFFDF53494840000000201A4A58C000000040B54A484000000060FD4858C000000040204B4840000000603D4958C0000000E0984E4840000000E0D84758C0000000E0754F484000000060224858C0000000A09D504840000000C0DD4658C0000000A0A250484000000060324658C00000008057564840FFFFFFFF724758C000000040FE58484000000040064758C000000060B35A4840000000009B4858C0000000E0F55C484000000040754858C0FFFFFF5FA55F4840FFFFFF7F714958C0FFFFFFBFB9604840FFFFFFBFE34858C000000040BC614840000000406B4958C0000000E0FD634840000000401C4B58C0000000809866484000000000854A58C0000000A0BA67484000000060864B58C0000000006468484000000060EF4A58C0000000E0016B484000000080394B58C0000000C0D76F484000000060D34D58C0000000A04677484000000080A94E58C00000000000804840FFFFFFFF051A58C00000000000804840FFFFFFDFB0D157C00000000000804840000000C014CA57C0FFFFFF1F0080484000000040B4C957C00000000095AF4840000000603CB557C00000000058AA48400000002096AB57C00000006047704840000000206EAC57C0FFFFFF9F89634840FFFFFF9F7CA457C0000000A05A5B4840000000408C9B57C000000000FC5A484000000080B29257C000000060975A4840FFFFFFBFC28E57C00000008075534840000000C0FF7557C0FFFFFFDFF84F484000000040007457C00000002042434840000000E0FA7157C0000000607D41484000000020E56057C0000000E064444840000000A0C55D57C0FFFFFF9F58464840FFFFFF9F495D57C0000000E0DF4B484000000040765357C0FFFFFFBF90514840000000E0D74557C00000008036504840FFFFFF1F983C57C0000000C07050484000000080A52E57C0FFFFFFBF2845484000000020112957C0000000402D454840FFFFFF1F142857C0000000A05F40484000000000B72C57C000000020563F484000000020372D57C000000080F03A484000000000D51F57C0FFFFFF5F5738484000000040321D57C0000000607D33484000000060481E57C0000000A0C52D484000000080AD1757C0000000A0451C4840FFFFFF9FB61157C0FFFFFFBF491F484000000040351357C00000004032264840000000C0A91157C0000000001C2D4840000000800D0857C000000020F52E4840FFFFFF3F3E0257C0000000A0842D484000000080AEFE56C00000006010204840000000E079F256C000000060661A4840000000608EED56C00000006033194840000000E007ED56C000000060B60E4840000000E064E456C000000060620D48400000008092E456C00000004097054840FFFFFF3F51CF56C0FFFFFF7F6B0A484000000000BBC156C000000080041948400000002052B756C0000000408D2048400000008091AF56C0000000C0550B4840000000804FA456C000000080970F4840FFFFFF7FA1A356C0000000E0E20B4840000000A04A8956C0FFFFFFFF720E484000000000B48156C0FFFFFF5F080B4840000000E0297F56C0000000A007034840000000809E7956C0000000200EFF474000000040F36F56C0000000A06703484000000020F56156C0FFFFFFFF3900484000000020096856C00000000010FF4740000000E0B86856C000000000D1FA474000000020F97F56C0000000E08EE94740000000E09BA056C0FFFFFFBFE2DA4740000000405DC156C00000000007BB4740000000A0FBDD56C000000040009047400000002040F356C0000000E0AC764740000000C0A60557C0000000E05E65474000000040B90D57C0000000A089554740FFFFFF9F631357C00000004054554740FFFFFF7F5F1257C00000000055544740000000A0761257C0000000E040354740000000E07A1257C0000000600D144740000000E0811257C0000000806109474000000080F01457C0000000A04907474000000060251657C000000040E602474000000060581757C0000000401602474000000000301B57C00000008045034740000000E0931D57C0000000A099FD4640000000A0852157C0FFFFFFFFC6FD4640000000C05B2357C0000000E0C4F94640000000E09F2A57C00000006037F54640000000C02F2D57C0FFFFFF7F0CF2464000000020F82E57C0000000E029EC464000000060E82F57C0000000402EEB464000000080D93157C0000000A0B6E14640000000E0563557C0000000408FDD464000000020073757C000000020F5DA4640000000E0A63857C0000000408FD24640FFFFFF7F1A3857C0000000A018CA4640000000C06D3557C0000000E01EC8464000000000C43057C0FFFFFF1F3BC84640000000A0962E57C0FFFFFF5F0DC6464000000080DA2B57C0FFFFFFFF2BBC464000000020E52957C00000000046BA4640000000E0432957C00000004041B84640000000C0812957C0FFFFFFDF9FB2464000000080D12B57C0000000807AAE464000000060422D57C0FFFFFFFFBBA84640000000C0C42F57C00000000019A6464000000060553057C000000040309B4640000000C0CA3057C000000000E4974640000000A0A92F57C0000000C006944640FFFFFF9FB12F57C000000000788E4640FFFFFFBFFA3257C0000000006788464000000080D13057C0000000E0D582464000000040153157C00000000022804640000000E0F82F57C0FFFFFF9FC4774640000000003D3057C0000000E01F75464000000000633157C0FFFFFFBF2373464000000060E63057C0000000C05E6E464000000060B13057C000000080EE6A4640000000608B3357C000000040835F4640000000202A2F57C000000020585B4640000000C0542857C0000000804352464000000040F62657C0000000201F4E4640FFFFFFDF932057C0000000E09F494640000000E0CD1557C000000060C4464640000000C07F1457C0000000E02F45464000000000FA1257C0FFFFFFDF003F464000000060EE0F57C000000060663A4640000000802E0D57C0000000601E384640000000A0D50557C00000002033354640FFFFFFDF38FE56C0FFFFFF9FA82E4640000000C013FC56C000000020692B4640000000C00BFB56C0000000A0A52846400000002005FB56C000000080E9244640000000C0DDF856C0000000C0F6204640000000404FF656C0000000007A1846400000004032F056C0000000E091114640000000C0BBE956C0FFFFFF3F910846400000004081E656C0000000E03A054640FFFFFFBF6AE456C0000000A07A04464000000040CFE156C00000006062044640000000A03FDB56C0000000002AFE4540000000C0E2D756C0000000C03EF9454000000000ADD256C0000000E071EC4540000000E00FD056C0000000C0E0E44540000000E08FD056C00000008078DC45400000004087D056C000000080B3D64540FFFFFF1FE7CE56C000000060A9CC45400000002063CF56C0FFFFFF1F3DC64540000000C04CCE56C0000000601BC04540000000E019E756C00000008015C04540000000E0BBEE56C000000000F3BF4540 Minnesota 27 0.51996527777777777778 0.53333333333333333333 0.51872871736662883087 0.53698224852071005917 0.49201277955271565495 0.52647058823529411765 0.62465373961218836565 0.54377880184331797235 0.56902002107481559536 0.62138364779874213836 0.57508342602892102336 0.51022395326192794547 0.52835051546391752577 0.51155327342747111682 0.59447583176396735719 0.62914321450906816760 0.67518248175182481752 0.67729083665338645418 0.71751412429378531073 0.79900881057268722467 0.73573407202216066482 0.69253012048192771084 0.71551334237901402081 0.68126825198164372132 0.69922353902738046588 0.71387163561076604555 0.70834982192322912545 0.66860676609676245908 0.68683274021352313167 0.73819113877700476016 0.72916666666666666667 0.73650034176349965824 0.73504273504273504274 0.73209798994974874372 0.75692307692307692308 0.74683916495148485739 0.77616522467206251744 0.77361899845121321631 0.75852205005959475566 0.77942831420211568760 0.77965752011553538271 0.79626719056974459725 0.80660377358490566038 0.81235738107776022468 0.87021310687389841372 0.85689123734037868780 0.85868213841690841276 0.85339251743817374762 0.86765381083562901745 0.86615541922290388548 0.85762464679609880594 0.82964868558565801109 0.81644428416877028489 0.80467020063074548749 0.79713905943414140135 0.81086461888509670080 0.80440228108511432074 0.79254416608443956483 0.77182005023978077187 0.73278627067105427584 0.73520176764740086057 0.74846648713345302214 0.76272195957264639095 0.75774401955648681683 0.74549016958313627458 0.76860343246429975108 0.76949322315084358469 0.78474545889101338432 0.77390718058208787504 0.78775499305778062587 0.78353689567430025445 0.78988681102362204724 0.77662669410345561626 0.79848171152518978606 0.81947261663286004057 0.81106001022017818658 0.79537237888647866956 0.76989419748499012379 0.76133877248284813647 0.77572990023474178404 0.77594053398058252427 +23 0106000020E61000000100000001030000000100000050010000FFFFFFBFB74656C000000060187A4240000000204B4856C000000080E76E4240000000E0A74A56C000000040F56B4240000000001A4B56C000000040296A424000000020854A56C000000040F7664240000000C00D4856C0FFFFFFBF6D654240000000C0074856C0000000E04E62424000000000B04956C0000000402861424000000040574B56C0000000006661424000000020904C56C0000000201C5D4240000000C0A34C56C0000000A04F5B4240000000C0BD4A56C000000000F655424000000080554B56C0FFFFFF9F95534240000000C0CE4C56C000000060CE504240000000A0714D56C0000000807B4A4240000000A0764F56C0000000C0DD484240FFFFFF9F235256C0000000C0A149424000000020A05456C0000000409E4F4240000000A0E85556C0000000207E50424000000060445756C00000000017504240000000A0ED5756C0FFFFFF3FDF4E4240000000C0C25A56C0000000205A414240000000A08A5A56C0000000C05540424000000080B45C56C0000000A06A3A424000000020225E56C00000000015394240FFFFFFBF7C5F56C0FFFFFF3F943B4240000000E0735E56C000000060D03F4240FFFFFFDFD35E56C0000000E099404240FFFFFFFF2A5E56C00000006045434240000000C0D35E56C0FFFFFF7F1D464240000000608F5F56C0000000209147424000000080F16156C00000006043484240000000C0976356C00000002064474240000000A05C6456C0000000E04C45424000000080496456C0FFFFFFFF65424240FFFFFFDF1F6256C000000000C23F4240FFFFFF7F066156C0FFFFFF3F643C424000000040E46256C00000006071384240FFFFFFBF476156C0000000E055334240000000C03C6156C0FFFFFF1F8F2D4240FFFFFFFFD96256C0000000A0402C4240000000A0C36656C000000080682D4240000000E0DB6756C000000020DA2A424000000040D56656C0000000C06D27424000000000B46256C000000060F3234240000000A0436256C0FFFFFF5FDC214240FFFFFF5FAA6256C000000080EE204240000000A08E6756C0000000C0D51E4240000000E0EA6A56C000000060A020424000000060736C56C00000000046204240000000A0856C56C000000060D21E424000000080506B56C000000060451C424000000080966756C0000000E084174240000000A0B96556C0000000607513424000000020B96556C0000000009D10424000000080B66A56C000000060B60C424000000020676B56C0000000C09E0A424000000000166C56C0000000404D03424000000040316E56C000000000FCFF414000000020A57D56C00000006098FF414000000020249256C000000040DEFE4140FFFFFFDF409856C0000000C0AAFE4140000000E02C9456C000000060BB0B424000000060399256C000000000D60E424000000080E09056C0000000A0330F4240000000A0078F56C0000000208C114240000000C0DC8E56C000000080A0144240000000C0078E56C0000000A016164240000000E0528A56C00000006035194240000000E0658856C000000020251B4240000000E0088756C00000000006214240000000E03A8456C0000000E0D9224240FFFFFF1F2F8356C0000000E07526424000000020548456C000000080A4294240FFFFFF7F368356C0000000E0692E424000000000558356C000000080F7304240000000A0218556C0000000A0DD324240000000207A8756C000000040D4334240FFFFFFDFEC8756C0000000A016364240FFFFFF9F808756C000000040193A424000000020C98856C0000000A08C3A4240000000409C8956C0000000C0F33E4240000000205C8E56C000000080123F42400000004039A556C0000000E0D73E4240000000E079B356C0000000409E3E4240000000A090C856C000000040753E4240000000A058DA56C000000060DA3E424000000080FBDC56C0000000A0C43E4240000000000FEC56C0000000A0D73E4240FFFFFFBF280857C000000040E53E4240FFFFFFBF5A0957C0000000A0EC3E424000000000772157C000000060D43E424000000000C23157C0000000A0B53E4240000000E0883657C000000060B23E424000000060045357C0FFFFFF7FCC3E4240000000A0005557C0FFFFFFBFBE3E424000000020296657C0000000C0B43E424000000060DE7657C000000000AF3E4240000000A02C8557C000000080D73E4240FFFFFF7F7DA757C0000000A0A23E424000000000BCA757C000000000D255424000000000C6A757C0000000A0BB614240FFFFFF9FB0A757C0000000209D7F424000000040B5A757C0000000E0B08742400000008099A757C000000060F1A942400000000096A757C0000000E02BAE4240000000C099A757C0000000003ED34240FFFFFF9F9EA757C00000008004D74240000000C074A757C0000000A0E2034340000000807EA757C0000000E022074340FFFFFFBF9AA757C0000000802D3243400000004095A757C0000000C0583C4340000000003AA757C0000000005D5E4340FFFFFFDF2EA757C000000020266B4340000000A0FAA657C000000080B7854340000000E0E7A657C000000020708E4340000000A076A657C0000000A013924340000000E031A757C00000002069934340000000E05AA957C00000004047944340000000C037AB57C0000000C06396434000000020BDAE57C0000000A0EB954340000000408CB257C000000060339943400000006084B457C000000020029B4340FFFFFFBF53B557C00000008081A143400000002060B857C0000000209DA443400000008013BA57C00000000059A94340000000804FBA57C00000002089AB43400000008079B957C0000000E0B8B04340000000803BBB57C000000060CDB04340FFFFFFFF53BD57C0FFFFFF5FAAB44340000000001ABF57C0FFFFFF5F40B84340000000C093C257C0000000C041BB4340000000E007C357C0000000601FBE4340FFFFFFBF83C657C00000008034C44340000000A0F5C657C0000000E0C4C74340000000606AC357C0000000A01BCB4340000000C091C357C0000000C016D0434000000080CBC157C0000000C0B9D44340FFFFFFDF9CBE57C0FFFFFFDFADD74340000000E089BD57C0000000A0B3DD434000000040FCBC57C0000000E045DE4340000000208FBB57C0000000600FDD4340000000A0F2B957C00000008006DD4340000000E02AB857C0000000C0A1DE434000000080BDB757C00000002087E04340000000E01DB857C0000000205EE14340000000C0FABA57C00000002001E14340000000E0D4BB57C00000006041E34340000000A0B6BB57C00000002032E44340FFFFFF7F8AB957C0000000A09AE5434000000040D9B857C0000000C0A0E84340000000207AB957C0FFFFFFFF06EA4340FFFFFFBF1CBB57C0FFFFFF3FA4EA4340000000C0EBBB57C0000000E0B8EC43400000002008BC57C000000000B3F24340000000E0BEBD57C0FFFFFF5F4EF34340000000A060C157C000000060D0F24340FFFFFFDF20C257C0FFFFFF5F5DF04340FFFFFF1F08C457C0FFFFFFDFEAEE4340000000206EC657C0000000E057EF4340000000809EC957C0000000403BF44340FFFFFFBF65CC57C0000000E080F443400000002045CD57C00000004016F84340000000C067CF57C00000000097F8434000000080BDD357C0FFFFFF9FECFF43400000006011D657C00000006032034440FFFFFF3FBED757C000000020AE03444000000040FAD857C0000000A099054440FFFFFFDF76DA57C0000000802806444000000060D3D957C0000000E0490A4440FFFFFF1F98D857C0FFFFFFDF340C4440000000A01FD957C0FFFFFFFFC50E4440FFFFFF9F05DB57C000000000DD104440FFFFFFFF7BDD57C0000000804516444000000020D9DD57C0000000004C1B44400000000080DE57C0000000A0091D44400000000000E357C00000006013224440000000E018E657C0000000C0A62744400000004061E957C0000000E090274440FFFFFF5F4CE957C0FFFFFF9F42294440000000E087E757C0000000E06B2A4440FFFFFF7F6BE757C0000000005A2C44400000002092E857C000000020ED2D4440FFFFFF3FBDE857C0000000E0BC324440000000607CEC57C0FFFFFF7F1F3E444000000020D2EB57C0000000E08F4144400000004019EA57C0000000E0E14344400000004069EA57C00000006084474440000000203AEB57C0000000406D48444000000020FAEB57C000000060D44744400000006046EC57C0000000A0164344400000002027EF57C0FFFFFFBF2444444000000040D7F057C0000000C05C464440000000E019F157C0000000E0654B44400000008077D857C0FFFFFF7FCB4A444000000020E6CD57C0000000A07B4A444000000040E7BA57C0000000A0E2494440000000C0EFA857C000000060B2494440000000200A9F57C0000000E07F494440000000003E8F57C0000000A01549444000000020248157C0000000A079494440000000204F7257C0000000800A4A444000000020036457C0FFFFFFDF574A444000000000AF5757C0000000404D4A444000000060724657C0FFFFFF7FCB4A444000000080ED2D57C000000080794B4440000000005C2957C000000040B44B444000000000201757C0FFFFFFFFBD4C4440000000005A0C57C0000000C0CE4C4440000000808EFC56C0000000A0DA4D44400000008075EF56C0000000400C4E444000000040E0ED56C000000080F44B4440000000A025EC56C0000000A0634A44400000006048EC56C0FFFFFF1F9C46444000000000D5E756C0000000E0344444400000000078E756C0000000609E404440000000E07EE556C000000060033E44400000000012E556C0000000205B3B4440000000801DE256C0000000804A3A4440000000E079E256C0000000C07A38444000000080E2E156C000000080AF37444000000020C3E156C0000000007F344440000000A003E056C0000000E0DA334440000000C05EDF56C000000080043244400000004085DE56C0000000400B324440000000C0B5DC56C0000000809A2F44400000000026DF56C0000000C0A127444000000080EEDF56C0000000202D204440000000406BE056C0000000A0A8194440FFFFFF3F08E156C0000000C038114440000000A041E056C0000000008A084440000000C02FDF56C000000080BC004440000000A09FDC56C0000000A018F94340000000808BDB56C0000000C0FEF5434000000080C7DB56C0000000206FF3434000000000DDDC56C0000000A04FF14340FFFFFF7FBFDC56C00000006078EE4340000000006ED856C000000000E2E6434000000020E6D756C00000006071E14340000000607ED756C000000000C1DC4340000000A054D456C000000020CCD743400000000002CD56C000000080CDCC434000000000FFC956C000000060BBC64340000000C0FDC556C0000000E0B3C34340000000E01EC456C000000080ABBC43400000006053C256C0FFFFFF7FE2B8434000000040AABC56C00000006046B34340000000806EB656C0000000A0DBAC4340000000C0E0B156C0000000A0FDA54340000000C03CAF56C000000040B89F4340000000A0DEAE56C0FFFFFF7FC49C4340FFFFFFDFF6AD56C0000000601299434000000000DFAD56C00000008075924340000000802FAC56C000000060FE8B4340FFFFFF1F49AD56C000000060728743400000004030AD56C000000060D6844340FFFFFFDFCEAA56C000000060B67743400000004024A856C0000000E0BD7043400000004080A456C0000000A0876F434000000080F2A156C00000004020724340000000E0119E56C000000060C67A4340000000C06F9A56C0000000A02D7B4340000000A0769456C0000000606376434000000000DA9156C0000000205D764340000000809C8F56C0000000A00E75434000000000808856C0FFFFFF1F306D4340000000603D8756C0000000C04C6A434000000060CA8756C00000002077664340000000C0A68856C0000000C08A64434000000020758A56C0000000E0F4624340000000A0948C56C0000000E0AA5C4340FFFFFF7FF18C56C000000080A5594340000000C0BF8B56C0000000A052544340000000E0C18B56C0000000601D4E4340000000A06B8F56C0000000000A48434000000000B89056C0000000C031444340000000A0029156C00000006064424340FFFFFF5F519356C0000000A0B336434000000020BC9556C00000004007324340FFFFFFBFF49656C0FFFFFF1FC32E434000000060A39756C0000000606A29434000000060589756C000000080FD1D4340000000C08C9556C0000000C02718434000000060899256C0000000405A15434000000080429056C000000040A30F434000000020488D56C000000040610B4340000000209F8856C0000000E0E706434000000040A38756C00000008021044340000000E0AE8256C0FFFFFF5F21FF424000000020B18056C0000000A012FC4240000000A0537D56C00000006058FB424000000080A67E56C0000000A0B8F4424000000020067C56C0000000C063F04240FFFFFF9FA27956C0000000A01DF04240FFFFFFDF797756C00000000029F24240FFFFFF5F1B7756C000000000E7F3424000000080827656C000000020D9F34240000000E09E6E56C0000000A0A5EB4240000000403A6C56C00000008003E7424000000040416B56C00000002059E4424000000040A76A56C0000000006BDF4240FFFFFF3F366556C0000000A061DA4240000000A0606156C000000020EFD8424000000020DB6056C00000000005D74240000000403A6156C0000000803FD3424000000000DB6056C0FFFFFFBFD6CE424000000020996156C0000000E035C9424000000080AA5F56C0000000E0F0BE424000000020085D56C00000000002BA4240000000605D5B56C0000000409CB4424000000020E35B56C00000002088AD4240000000E0FF5D56C0000000C071AB424000000080096056C0FFFFFF1F2BAA424000000080E36056C00000000009A7424000000080E36056C0FFFFFF1F61A3424000000080555F56C0000000A0C4A04240FFFFFF9FC75D56C0000000407AA0424000000040F75D56C0FFFFFFBFB49C4240FFFFFF7F315C56C00000002029954240000000801F5B56C0000000E08F914240000000C0515856C0000000C0AE8C424000000040825856C0000000A04C86424000000020E75356C0000000403D814240000000E0185256C000000000E67F4240000000A0C45056C0000000A01C81424000000060E95056C0000000C08C83424000000020D25356C0000000E0CB87424000000020695356C0000000E0ED8A4240000000E0305256C0000000E0AD8B424000000040E65056C0000000E0268B424000000020364F56C00000008057854240FFFFFF7F714D56C000000060B583424000000080624C56C0000000804E7E4240FFFFFF9F4F4856C0000000807A7E424000000080DB4656C0000000201D7D4240FFFFFFBFB74656C000000060187A4240 Missouri 29 0.53906250000000000000 0.54202898550724637681 0.55732122587968217934 0.53994082840236686391 0.53354632587859424920 0.53970588235294117647 0.58171745152354570637 0.53686635944700460829 0.53530031612223393045 0.59748427672955974843 0.56062291434927697442 0.50535540408958130477 0.54982817869415807560 0.51732991014120667522 0.60451977401129943503 0.66791744840525328330 0.68734793187347931873 0.67785998861696072851 0.69152542372881355932 0.75770925110132158590 0.73518005540166204986 0.68771084337349397590 0.70284938941655359566 0.69211514392991239049 0.70984879444217409072 0.71469979296066252588 0.71943015433320142461 0.69297926518734085122 0.69359430604982206406 0.74844379348224093739 0.75070621468926553672 0.73684210526315789474 0.72813938198553583169 0.73178391959798994975 0.74676923076923076923 0.74478094678035871802 0.76416410828914317611 0.74728962312854930305 0.73110846245530393325 0.75849651136619401305 0.73468124613162781102 0.75500982318271119843 0.77490566037735849057 0.77988414955239599789 0.79105912514020189072 0.77528254806986643182 0.79196021549937836718 0.79974635383639822448 0.80245638200183654729 0.79621676891615541922 0.79418466867195333151 0.75488383310555510893 0.75420122610890732059 0.74045494195799503456 0.74156592189379074625 0.73720136518771331058 0.73911421414485956404 0.72387463818744385667 0.69650605160995661110 0.67005456741783646436 0.66220878396712796062 0.66393626570915619390 0.69091315191899638909 0.68245154531168150864 0.68559556786703601108 0.69654788418708240535 0.69158293423482643128 0.69009918738049713193 0.68131250702326104057 0.67828153369646480829 0.67114503816793893130 0.67531988188976377953 0.66595709453865985477 0.68468146878941481640 0.70055191282607670173 0.68535181852518385212 0.67296159244608906469 0.65229890677253437115 0.64079362136102354905 0.65608494718309859155 0.66569326456310679612 +8 0106000020E61000000700000001030000000100000031010000000000004C3254C0FFFFFFDFF0C83C4000000020CF3054C0FFFFFFBF6EBC3C40000000E0443554C0000000E031C93C40FFFFFF9F753654C00000000012C93C40FFFFFF9FD52F54C0000000E011663C40000000A0841F54C0FFFFFF3FB0DE3B40000000009D2054C0000000C04DD33B4000000060DA1E54C0000000606DD83B40000000E0B31954C0FFFFFFFF09B43B4000000000E31654C0000000602B8E3B40000000005A0F54C0000000C0AA433B40000000402E0C54C0000000A0732F3B40000000A0590E54C0FFFFFF3FF9363B40000000208D1254C000000020B33D3B40000000E0ED1454C0000000E0783F3B40000000009A1254C00000004068363B40000000202D0E54C000000020D8333B40000000207C0954C000000060D61B3B4000000060CC0554C0FFFFFFFF44F93A40000000A0590754C00000008037F93A40FFFFFFFF450354C0000000E0FECB3A40000000806F0554C00000002076533A4000000060310854C0000000A027FA3940000000805F0C54C0FFFFFFBF78C23940000000C04E1354C0000000C0059D394000000040221554C000000040587D394000000040571354C0000000809B663940000000C0B01A54C0FFFFFF9FDF3F394000000020F21A54C0000000201B31394000000080552354C00000006040363940000000206B2C54C0000000C0E226394000000040D93654C0000000406B2F394000000020783E54C0000000204E21394000000040A14754C0000000C03F223940000000E07F4954C0000000C00E2A394000000060C64B54C000000020BB44394000000060024954C0000000A003523940000000C0C04054C0FFFFFF9FCA36394000000000E13C54C000000040BF333940FFFFFF3F973A54C0FFFFFF3F0F3F3940000000005E3E54C0000000C072523940000000802F4954C0000000A07C653940000000808C5054C0FFFFFF5F43AE3940FFFFFF3FCF4C54C000000020C5B5394000000000695054C0000000407FCD3940000000E0FA6154C0000000600EEA3940000000A0FB6D54C00000002057EC3940000000402D6D54C0000000C0F2FF3940000000C0DF7254C0000000E05C1C3A40FFFFFF5F197454C0FFFFFFFFBA483A40000000E05E7654C000000020F3543A4000000060527754C0000000206F703A40000000204D7C54C0000000C09A773A40000000C0F87D54C0000000E05B843A4000000020747B54C000000040DD883A40000000209B7854C0000000E083A43A40000000008B7154C000000040BDB53A40000000406D7954C0000000C0D5A93A4000000000757C54C0000000C0E48C3A40000000E0628154C0000000203A863A40000000C05D8554C00000004027B73A40000000A0618454C0FFFFFF7F41C43A4000000040538354C000000040CADD3A40000000A0468654C0000000E0E3EB3A40000000E0767F54C0FFFFFFBF74F63A40FFFFFF3FAE7E54C00000004006083B40000000008E7F54C0FFFFFFFF05083B4000000000D38054C000000020E9F93A4000000040D18954C000000040CDEF3A40000000407D9054C0000000E00A013B40FFFFFF3F129254C00000006033063B40000000407F9054C0000000C060FF3A40000000C04E8B54C000000080D1E93A40FFFFFF5FDB8954C0000000402FCA3A40FFFFFFBF8F9254C0000000A07BD93A4000000060C29654C0000000A0D4F23A40000000A07E9954C0FFFFFF9F88FF3A40000000409B9754C0000000E095F23A4000000000019654C0000000203EE73A4000000080559854C0000000C06BF23A400000008074A054C000000000963C3B400000000001A254C0000000208C423B4000000040F0A054C0000000C0CA353B40000000207DA454C0000000E021463B40000000601BA254C0000000C0DB543B40FFFFFF1F35A454C0FFFFFF7FE8623B4000000020E9AB54C00000002039793B4000000080E6A854C0FFFFFFFFCD803B40000000E09FAA54C000000000477E3B4000000000A4AB54C0000000602C863B40000000C0429F54C0000000C04D7A3B40FFFFFF9FEBA054C000000000FD823B40000000E0599B54C000000060C5853B4000000020D9A454C00000000013833B4000000080E4A854C0000000804C893B40FFFFFF5F77A454C0000000006A8D3B40000000007CA354C000000020F9943B400000008019A854C0FFFFFF9F0C8E3B40FFFFFFFF92A254C000000060989B3B40FFFFFF1F5FA354C0000000C0C9A43B4000000060E49954C00000002094CA3B4000000040899954C0FFFFFF3FEAE73B4000000000849D54C0FFFFFF5F9AF03B40000000A0EE9E54C00000004057D23B40000000A0B0A054C0000000C0B7D43B40000000E081A254C0FFFFFF3F78EF3B40000000E053A954C0000000204F073C40FFFFFFBFA7A954C000000000C2013C4000000080B4AC54C000000060BE0B3C40000000C010AB54C0000000A09D023C40000000A0DAAC54C0000000409DF93B400000006046A954C0000000805EF73B400000004071AE54C0FFFFFF1FB5F03B40000000E01FA454C000000020CFE03B40000000E0ECA754C00000004025D93B400000006044A954C0FFFFFF9F25B73B40000000A06CAB54C0000000408DB43B40FFFFFF1FD1B254C00000006050D43B40000000209DAE54C0000000E0ABB73B40000000806FAF54C0000000E080AF3B40000000A00BB654C0FFFFFFDFAFD93B4000000060E0B154C000000080382C3C40000000201DAB54C0000000A0A06D3C40FFFFFF5FB8A854C0000000A046B13C40000000A09EAB54C000000020E1CE3C40000000A0BFA854C00000008063D03C4000000040C1A854C0FFFFFF5F6BE23C40000000005AB054C0FFFFFF1F27023D400000004059B354C0FFFFFF9FA6273D40000000A05CC254C000000060DC2D3D40000000C0BDC454C001000000F2433D40000000601CC954C0FFFFFF3FA64C3D4000000040FCC654C000000020F4533D40000000A02FCB54C0000000C0EB573D40000000A001CF54C0FFFFFF1F076F3D40000000C054D854C00000006007853D4000000060EDD954C0000000405CAB3D40000000C03CE354C0FFFFFF9FB2BC3D4000000040E0E954C0000000A026E93D40000000E032FE54C000000040C8133E4000000080C80455C0000000A088193E40FFFFFF3F780955C000000060DE143E4000000060EC0E55C0FFFFFFBF9F1B3E4000000060A81655C000000080C5113E40000000C0121755C0000000C028FA3D40000000C0041C55C0000000A0D8FD3D40000000A0B91B55C00000004097F53D40000000E0B21555C0000000406AF23D4000000080351655C000000020F3E83D40000000A0BC1D55C000000080EEED3D4000000080EB3655C00000006024BF3D40000000406E3B55C0000000601BC73D40FFFFFF7F413F55C0000000E03EB83D4000000000C04D55C0000000608BB33D40FFFFFF7F5B5755C000000060CCAE3D4000000040385A55C000000060CCC63D4000000020815A55C000000060E4DC3D4000000020285955C0000000800DE03D40000000A0C45955C0000000C04DCB3D4000000080EB5655C000000020E8B03D40FFFFFF7FA05355C00000006095B33D40FFFFFF1F3D5355C00000002041CF3D40000000C0A15855C0FFFFFFBF70EC3D40000000803A6855C0000000C0A6173E4000000000646455C0000000801C193E40000000002C5E55C0000000808B053E40000000E0905A55C0FFFFFF7F06083E4000000080AD5B55C0FFFFFF3FA90B3E40000000607F5855C0FFFFFF3F29063E40000000C07E5855C0FFFFFFBF3C093E40FFFFFFFF535955C0FFFFFFFFF00E3E40000000A0B85D55C0000000C00B0D3E4000000020E86155C000000000A9213E40FFFFFFFF6E6D55C0000000E0B62D3E40000000E0A16A55C0000000A061403E40000000806A6655C0000000603E403E4000000080736455C000000000914F3E40000000A0E66D55C0FFFFFFFFCD433E40000000802D7055C0FFFFFF7FF64B3E40000000606E7655C001000080B8473E40000000E01E7555C0000000808C3B3E40FFFFFFDF827055C0010000008F3A3E40000000607E6E55C0FFFFFF7FEE203E40000000402C7F55C0FFFFFFDF33463E4000000000CD9855C00000008030633E40000000E057A055C0FFFFFFDFE6683E40000000A0D79855C0FFFFFF5FE3673E40000000A05D8F55C0FFFFFFFF51663E40FFFFFFBF768F55C000000000A76D3E4000000040578755C0000000E0B6623E4000000020DF8755C0000000A01F6D3E40000000000F8E55C000000020D77C3E40000000E0D39855C0000000A043763E40000000E0F89C55C0FFFFFF1F44803E40000000E00EA755C000000060676C3E40FFFFFFBF95B255C0000000A0F26A3E40000000E060CC55C0000000C0E25A3E40000000E0AEBB55C0FFFFFF3F9F763E4000000080ECC055C0FFFFFF1FA5833E40000000601EBF55C0000000C019973E400000006041C155C0FFFFFF7F57963E40000000806FC455C0000000404B733E40000000E0F8C755C00000004081903E40FFFFFF3FFBCA55C000000000BB8E3E40000000A03FCA55C000000080FF763E400000000088D155C000000080705B3E400000008024DB55C000000000CF523E40000000A031DA55C0FFFFFF1FEC663E400000008032D655C0000000C0696E3E40FFFFFF1F3CD655C0FFFFFF1FFC743E4000000040CEDA55C0000000E0437B3E400000000081DC55C0FFFFFF5FFC873E400000008003DB55C0000000A0698E3E40000000E02BD955C001000020B29E3E40000000C083D955C000000080F6AA3E4000000080CFDA55C0FFFFFF7F4FB13E400000004073DD55C000000000A3B43E4000000020B4E155C0010000C090BF3E40000000C0AEE255C0FFFFFFFF16C93E40000000606BE755C0FFFFFFBF1DD93E40000000E00BE855C0FFFFFF7F70E03E40000000A0C0E555C0FFFFFFFF44F43E40000000604FE655C001000080A0003F400000000071CA55C000000000C3003F40000000C0E1B155C0000000A07DFF3E40000000E0EBAC55C00000002077FF3E4000000000939855C0000000A0C9FD3E40000000409E8B55C000000040BBFE3E40000000600A8255C0000000E03EFE3E4000000080255F55C0FFFFFF7F35003F40000000800B5F55C0000000C035003F40000000801B4055C00000000046003F4000000080034055C000000080A8FA3E4000000020503E55C000000080A6F63E4000000080053E55C0FFFFFFBF49ED3E40FFFFFFBF053C55C0000000A00FE53E4000000000543B55C0FFFFFF7FC3D83E4000000060B03B55C0FFFFFF5F6CCD3E4000000080533B55C001000040A1C63E4000000040963A55C0FFFFFF7FFFC03E4000000000DB3855C0FFFFFFFF62BE3E40000000A0563755C0FFFFFF1FDDB63E40000000C03C3755C0FFFFFF9F64B63E4000000080F81755C0FFFFFF3FA3B13E4000000060081255C000000080B2B03E40FFFFFFBFD70455C0FFFFFF5FAAAD3E40000000C00D0055C000000020E3AC3E40FFFFFFBF3AEF54C0000000E008A93E40000000C0FCE654C0000000A08CA63E4000000000E3D354C00100002054A23E40000000E04AD354C0FFFFFF5F11A23E40000000207CC854C000000060029F3E400000008093AC54C0000000E072983E40FFFFFF9F51A554C0000000E0BD963E4000000080A69D54C0FFFFFF9F1C953E4000000040629A54C0FFFFFFBFBB933E4000000020288E54C0000000C01C913E40FFFFFF7F448F54C0FFFFFF9FFD873E40000000C00C8E54C000000080B3803E4000000020C28C54C001000060627D3E40000000A02A8D54C0FFFFFF5F726C3E40000000A08B8B54C0000000404F5E3E40000000208C8A54C0000000406E5C3E4000000040638354C000000020125D3E4000000020778254C000000060F0603E4000000000F68254C0000000A00F6F3E40FFFFFF1F428254C00000000051713E4000000080738154C0FFFFFF3F3C7A3E40000000C0618054C0000000001D923E4000000060E28054C0FFFFFF5F34993E40FFFFFF7FE28254C0000000C092A63E40000000000D8254C000000060DEC13E4000000020D78054C0FFFFFFFF96C33E4000000040018154C0FFFFFF9F84CA3E40FFFFFF3FD37E54C0FFFFFFBF32C73E40000000C08C7D54C0FFFFFFDFB7CB3E4000000020017D54C000000080F8D13E40000000006A7C54C0000000C0F2D23E4000000080177A54C0000000A030D03E40000000408E7954C0000000C0FCD33E4000000000E27754C00000008081CC3E40000000606A7354C0000000A0B0C93E4000000060B87154C0FFFFFFDFE7C23E4000000020817054C000000040F7C43E40FFFFFF7F286F54C0000000E07EC33E40FFFFFF5FE26D54C0FFFFFF1FC1BE3E4000000040216954C0010000A095BA3E40000000C0336854C0FFFFFF1F46BB3E4000000080B76654C00000006051B73E40000000807C6654C0000000C082B93E40000000805A6254C0000000C0D2B43E40FFFFFF7FD76154C0000000A0A1B83E4000000020EF5F54C0FFFFFF9F2E993E40000000E02B6154C000000060548E3E40000000404B5D54C0FFFFFFBF5B743E40FFFFFF7FC85E54C0000000605B613E4000000080615C54C0000000E0605B3E40000000A0B55B54C0FFFFFF3F1D3F3E40000000C0635354C0000000A0ADE93D40000000C0445454C0000000E038D43D40000000A09D4F54C000000080D9BC3D4000000080F94E54C0000000C034AB3D40000000E00B4754C0FFFFFF7F136E3D40000000C0394654C0010000C0FE593D40000000004C3254C0FFFFFFDFF0C83C4001030000000100000012000000000000005FB555C0FFFFFFFF47663E40FFFFFF7FA9B255C0000000805D673E4000000060BCA755C0FFFFFFFF126A3E40000000403AA255C00000006016683E40000000E075A155C0FFFFFF7FA4663E40000000A017A255C0FFFFFFFFA7643E40FFFFFF9F46AF55C00000000001673E40000000A0C0B255C000000000EF633E40000000806AB555C0000000E007633E400000002083CF55C0000000403C523E40000000C017D155C0000000403C523E400000002095D255C00000000020553E40000000C04FD255C000000040EE563E4000000060F0CC55C0010000A0AA553E40000000A0EBC855C0FFFFFF3F005A3E40000000200DC355C000000000845E3E400000006077BB55C0FFFFFFFF95613E40000000005FB555C0FFFFFFFF47663E4001030000000100000008000000000000A01B2F54C000000020E0C93C4000000020DE3054C0FFFFFF7F61D23C40FFFFFF5F503454C0FFFFFFDF26E53C40000000200B3954C0000000805C033D4000000020A83A54C0000000C051123D40FFFFFF7F8E3954C0FFFFFFFFAD0F3D40000000E0353754C0000000E0C9FD3C40000000A01B2F54C000000020E0C93C400103000000010000000F000000000000E0782E54C0FFFFFF9FBAC83C40000000601D2554C00000006002973C40FFFFFFDFAC2154C0FFFFFF3F93763C40000000E09F2554C0000000E0B2683C4000000080CC2754C0000000A091693C40000000A0F82454C000000080628C3C40000000E05B2654C0000000C0889A3C40000000E0412A54C0000000004E9E3C40000000804B2C54C0000000208C963C4000000080F63154C0FFFFFF3F629E3C4000000060503254C0FFFFFF5FFCAF3C40000000E00D2F54C0FFFFFF1FD3B43C4000000060463054C0FFFFFFBF92BC3C40000000200D2954C00000000036A83C40000000E0782E54C0FFFFFF9FBAC83C400103000000010000000800000000000020162C54C0FFFFFF7FDC943C4000000080DB2954C000000000C2993C40000000C0002754C0FFFFFF3FC3923C40000000E06D2A54C000000060586D3C40000000A0B52A54C0000000C0324D3C4000000060C52654C0FFFFFFFF07253C4000000040292E54C00000002097623C4000000020162C54C0FFFFFF7FDC943C400103000000010000000700000000000040898654C000000000F2953A40000000004C8454C000000000497F3A40FFFFFFBF3F8654C0000000A05B7E3A4000000080B28854C0FFFFFF5F6CA43A40FFFFFF5FBE8B54C000000080DEAE3A40000000E03E8854C0000000E07EB13A4000000040898654C000000000F2953A4001030000000100000007000000000000C0FA0F54C0FFFFFF7FC35A3940000000E0C61654C0FFFFFFBF67283940FFFFFF5FA22554C0FFFFFF3FBAF43840FFFFFF3F9C1654C0000000E00C36394000000040271554C0FFFFFF7F8044394000000020281754C000000080CD4B3940000000C0FA0F54C0FFFFFF7FC35A3940 Florida 12 0.44965277777777777778 0.45410628019323671498 0.45175936435868331442 0.47189349112426035503 0.46006389776357827476 0.51176470588235294118 0.52077562326869806094 0.51843317972350230415 0.51317175974710221286 0.57861635220125786164 0.55061179087875417130 0.50827653359298928919 0.52319587628865979381 0.50513478818998716303 0.63339610797237915882 0.69668542839274546592 0.71411192214111922141 0.66533864541832669323 0.65932203389830508475 0.66134361233480176211 0.66925207756232686981 0.62843373493975903614 0.62777023971053821800 0.61493533583646224447 0.64037597057621577442 0.64679089026915113872 0.66204986149584487535 0.65150963986904328847 0.65338078291814946619 0.69095569388502380081 0.70974576271186440678 0.69138755980861244019 0.67028270874424720579 0.66363065326633165829 0.67692307692307692308 0.69038518082916789180 0.69718113312866313145 0.69308208569953536396 0.69344457687723480334 0.73126266036461850101 0.75489993810604497627 0.78703339882121807466 0.80867924528301886792 0.82552220466912410040 0.83880788335202691876 0.82430647291941875826 0.81433899709904682967 0.80862396956246036779 0.80463728191000918274 0.80991820040899795501 0.80931546805213745329 0.80786236835758501487 0.80742877749729534800 0.79104878212440448232 0.79986075068042281157 0.78293515358361774744 0.78372328518893567127 0.76968759357221279569 0.74971454670015985385 0.73282792518848669138 0.73826413924099701516 0.74263165769000598444 0.75155418233257640621 0.72152959664745940283 0.73143706506317140734 0.73169134023319795624 0.73596894857107083607 0.73542065009560229446 0.72260928194179121250 0.71905372209761828474 0.70687022900763358779 0.69254429133858267717 0.68241916790841789068 0.70200613978724923253 0.71411387329591018444 0.72123353107156346508 0.73284845391518863511 0.72231240099348757163 0.70055627665492304840 0.69336854460093896714 0.69335937500000000000 +38 0106000020E610000002000000010300000001000000F3000000000000E09F7054C000000060FD98404000000060487154C0000000A05B9B404000000080EB7154C0000000E0CD9B4040000000804F7354C000000080989A4040FFFFFF3FE67354C000000040F79C404000000000AA7654C000000040269F4040000000C0907554C00000000057A14040000000A0C27554C000000000F7A2404000000040E97454C0000000C077A24040000000A0187754C0FFFFFF5F02A6404000000000C77554C0FFFFFF1F73A74040000000C0677754C00000002063A8404000000060217854C0000000C041A7404000000080317954C000000060E4AA404000000060627A54C0000000C080AA404000000040577A54C0000000A0B7AC4040000000C0F57B54C000000080D6AC4040000000A07E7C54C0000000C041B0404000000020437B54C0000000802EB0404000000020317C54C0000000203AB44040000000807A7A54C0000000A028B54040000000205B7B54C000000000CFB74040000000E0AC7A54C0000000C0C0B9404000000060F07B54C0000000E045BC404000000000CA7E54C0FFFFFFFFCBBE4040000000E0C97F54C0000000009BC24040000000A04C8154C0000000C0F0C44040000000207A8254C00000004010C64040000000C0368454C0FFFFFFFF6FC94040000000C0758754C0000000401DCC4040000000A0E78854C0000000C000CC4040000000E0B48A54C000000000BBCE4040000000604F8C54C0000000A0D5CF404000000020B88D54C0000000001ED74040000000200B8F54C0000000C058D84040000000400C9154C0000000C077E1404000000000909354C0FFFFFF9F29E44040000000E0EB9354C000000080E2E64040000000607E9654C000000000EAEA4040000000406A9754C000000040FEEA404000000000029954C00000006050ED4040000000E02D9B54C0000000C018EE4040FFFFFFDF3B9D54C0000000A065F04040000000C021A154C00000000028F74040FFFFFF9FE9A454C000000000C6FA404000000060B8A454C00000002001FC40400000008029A654C0000000E0B30141400000000099A654C0000000206B044140000000A045AA54C0000000E0DA0D41400000006019AF54C000000040B81541400000000089AF54C0000000004C1A4140000000E085B054C0000000A0DB1D414000000080EAB054C000000000F3234140000000E01CB254C0000000202C25414000000020C2B354C0000000607E2B4140000000808AB554C0000000007B2F4140000000E007B654C000000020CF3441400000004090B754C0000000608D3A4140000000A0EEB954C0000000402B3D4140FFFFFF9FDABE54C000000080FA3C41400000006084C054C0000000C0403C4140FFFFFF1F87C354C000000000B73E4140000000C010C554C0FFFFFF5F6A424140FFFFFF7FEDC654C00000002078444140FFFFFF9FCCC854C0FFFFFF9FAC484140000000402ECA54C000000060D2494140000000608FCA54C0FFFFFFBFA64C4140000000C02CCF54C0000000207E4E41400000000023D354C000000040D854414000000000C1D554C000000080BB5641400000008073D654C0000000A0D25A4140000000C06DD654C0000000C0175D4140000000E0B4D454C00000002046604140000000E0A8D454C000000060C16441400000004062D354C0000000A018674140000000804BD154C0FFFFFF7F516841400000002025D154C0000000E0686B4140000000E008D054C0000000A0786B41400000000007D054C000000040CF6C41400000008017CF54C0000000A05F6E41400000008090CF54C0000000006270414000000040E8CE54C000000060D66F4140000000200DCE54C0FFFFFFBFCD7141400000006008CD54C00000008029714140000000E0AACB54C0000000208D7441400000008024CA54C0000000A07275414000000000EEC954C0FFFFFFFF51774140000000E0CFC854C0000000E016774140FFFFFF7F0CC854C0000000E068784140000000203FC754C000000000D0774140FFFFFFFF27C854C000000000267A414000000020C7C754C0000000A0FB7A4140FFFFFF1F67C754C000000040307A4140000000E06FC654C000000020F67D414000000060CDC654C000000020098041400000004077C054C0000000201983414000000000D0B854C00000006016874140000000C05DB154C0FFFFFF1FEF8A414000000080A1AC54C000000080AD8B4140000000C008AC54C0000000C0848C414000000040E8AB54C0000000208C8F414000000040E3A954C0000000604A8F4140000000607BA454C00000008022934140000000A093A154C0000000A0C8934140FFFFFF1FDF9D54C00000004035964140000000E0069C54C000000020B495414000000060EF9854C0FFFFFF9FA79A4140000000A0C49754C00000008063974140000000400C9754C0000000E06A97414000000080749654C0FFFFFF7FA9984140FFFFFF7F879454C0000000809397414000000020D09154C0FFFFFFFFF5984140FFFFFF7F738D54C000000060B8984140FFFFFF1F2C7E54C0000000A019984140000000C0B77754C00000006070974140000000A0FB7054C0000000405B974140000000E02D5754C000000060D894414000000020A95454C000000000F6944140000000C0274354C0000000606593414000000080234354C0000000C0E590414000000000D24154C0000000007E8D414000000000324454C00000008082884140000000808D4254C000000000C384414000000000613B54C0000000C0F58C414000000000433954C000000060A587414000000040C53554C0000000204280414000000020483254C0000000E066784140FFFFFF3F343354C0000000607668414000000060F02354C0000000605968414000000060D11454C0000000204A68414000000080E0FA53C0FFFFFFDF69674140000000C0E6EB53C0000000A01267414000000060B8EA53C0000000207C664140000000A02EDD53C0000000E029514140000000009EDC53C0000000603D4F4140000000A0C4C453C000000040FC26414000000040F7A953C0FFFFFF1F6EF94040000000201AA553C0FFFFFF1FE5F04040FFFFFFDFDCA753C0000000E0C9EE40400000004084A553C0000000E03AED404000000080BAB653C000000020ACDB4040000000600FC053C00000006046C94040FFFFFF1FC0C753C0FFFFFF3F1EB74040000000E09FC953C00000006096A840400000006022CA53C000000060CFAB4040FFFFFF3F59D153C0000000E0FFA54040000000E0EFCC53C0FFFFFF9F7D974040000000E0B4CE53C00000006017924040000000A055D653C000000060D19340400000006013D353C000000000C49140400000004070D253C000000040618D4040000000204BDA53C0000000E0C0814040000000C048E553C000000020078240400000002084E753C0000000C08A7D4040000000609FE553C00000004064764040FFFFFF5FD9E653C0000000C0147340400000004029F053C0000000C0A36540400000006017FA53C000000020306540400000002034F353C00000002003774040000000C011FA53C0000000C0FA6D4040000000608CFB53C000000080FF7440400000008096FD53C000000000BE734040000000E0AEFC53C0FFFFFF7FC26740400000000014F953C0000000E0EE5D4040000000A061F953C0000000E0AF564040000000A0CBFF53C000000080844D4040000000C0800D54C000000080DC474040000000009D1254C0000000407F404040000000003B1654C0000000A081414040000000E0061954C000000040DE53404000000080A21A54C000000020A1554040FFFFFF3F991954C0000000609C404040000000C0C61E54C0000000C04F414040000000803C2354C00000004061474040000000E07F2254C0000000203D414040FFFFFF9F612954C0FFFFFF7F64424040000000E0322954C000000060C73F4040000000A01E1F54C0000000C025374040FFFFFF9F591C54C0FFFFFFBFC82F4040000000E0791D54C000000040C4284040000000A0132854C000000040E522404000000060672B54C0000000208A244040000000207F2F54C0FFFFFFFF142E4040000000E02B3354C0000000208F3C4040000000C03C3254C0000000609F404040000000002A3554C0000000600F424040FFFFFF9F823754C00000004029444040000000409C3454C0000000204433404000000080F73154C0000000E0BB1F4040FFFFFF3F283954C000000000B308404000000000B94654C000000080780D404000000040A24754C000000040070F4040000000405A4754C0000000C05E18404000000080764954C000000020B41C4040FFFFFF5F7E4954C000000040F620404000000020F34754C0000000A062234040000000A0734854C0000000E08C2A404000000060174954C000000040942C4040000000E0164A54C000000080552B404000000080794B54C000000040902F4040000000406B4B54C0000000607D31404000000060CE4C54C0000000C0C635404000000040804C54C0000000A0703B404000000060214F54C0000000809D42404000000080295154C0000000E05144404000000000915154C000000040FE46404000000060255354C0000000009748404000000000CC5554C0000000801C494040000000A0815654C0FFFFFF3FAB4A4040000000E07A5754C000000080784A4040000000C0735754C0000000C0494B404000000020A55854C000000020404C4040000000C0625A54C0000000400E504040000000E0935954C0000000404853404000000040125A54C000000040E0574040000000E0EC5A54C000000060C759404000000060195A54C000000000EE5E4040000000E0155B54C000000060F45F4040FFFFFFBFA35A54C0FFFFFFFFD6604040000000E08B5B54C0000000409B644040000000C0F15A54C00000002098674040000000008D5B54C000000060FA68404000000000255B54C000000020726A404000000020AE5B54C0000000E0B66B4040FFFFFFBF265D54C0000000C0156C4040000000805B5D54C0000000C0846F4040FFFFFF9FEF5E54C0000000801E704040000000E0C55D54C000000080E3724040000000C08A5E54C0FFFFFF5FDE72404000000080AA6054C0000000603979404000000000A46054C000000000467A404000000060E05F54C000000080D17A4040000000C08B5F54C0000000A098804040FFFFFF1F926054C0000000209E814040000000C0E76154C0000000609A854040000000A00F6354C0000000609285404000000060DA6354C0000000C0C3874040000000C0246654C0000000A00A894040000000C01C6754C0000000803A8B4040000000A0156D54C000000080B78F404000000000656F54C0000000808092404000000060DE7054C000000040B8954040000000E09F7054C000000060FD9840400103000000010000000500000000000080263154C0FFFFFFBF1421404000000040082E54C000000000D1224040000000A0AC2A54C000000000261C4040000000A0713454C0000000E05F0D404000000080263154C0FFFFFFBF14214040 South Carolina 45 0.23524305555555555556 0.23478260869565217391 0.23269012485811577753 0.23520710059171597633 0.27955271565495207668 0.31029411764705882353 0.31717451523545706371 0.29723502304147465438 0.28767123287671232877 0.31446540880503144654 0.30700778642936596218 0.30185004868549172347 0.33848797250859106529 0.34980744544287548139 0.40677966101694915254 0.45778611632270168856 0.45802919708029197080 0.44393853158793397837 0.44802259887005649718 0.50165198237885462555 0.48365650969529085873 0.44578313253012048193 0.50429669832654907282 0.49895702962035878181 0.50388230486309767062 0.48281573498964803313 0.48634744756628413138 0.45834849036013095671 0.45765124555160142349 0.48187477114610032955 0.49152542372881355932 0.49111414900888585099 0.49243918474687705457 0.50314070351758793970 0.51353846153846153846 0.52543369597177300794 0.54646943901758303098 0.56169334021683014972 0.55780691299165673421 0.57843799234751294171 0.58324736950691149164 0.60196463654223968566 0.61773584905660377358 0.63243812532912058978 0.64556962025316455696 0.65448407456333480112 0.65202376018787125294 0.66670894102726696259 0.65243342516069788797 0.64764826175869120654 0.64205633032540333607 0.62657769917195916070 0.62394518571943743238 0.60866939542374018654 0.61871004493955313627 0.62059158134243458476 0.62175558279592815648 0.61173769837309112686 0.59630052523407170587 0.58503769733827633607 0.57502810404310578749 0.60031418312387791741 0.61084018910769459852 0.59944124323380478435 0.60147962975474630093 0.61201362504912878292 0.60954080195323504554 0.60955425430210325048 0.60076974941004607259 0.60194382142475702232 0.59910941475826972010 0.59844980314960629921 0.58825894652191395255 0.60158016229980247971 0.61028822114250672202 0.60114643737919082850 0.60263281017396112458 0.58650969041519175484 0.57404042277025774152 0.57786825117370892019 0.58470494538834951456 +34 0106000020E6100000010000000103000000010000003C010000000000401A9C57C000000020EB764140FFFFFFFF869C57C000000000255E4140FFFFFF3FF49C57C0FFFFFFDF10414140000000E0889D57C0FFFFFFFF2B194140FFFFFFDFF99D57C0000000A037F84040FFFFFFBF7E9E57C0FFFFFF3FE4D04040000000000AA057C0FFFFFFFFBFCF404000000000ADA057C000000060BED04040000000809AA157C0000000A07DCF4040000000C026A157C0000000204ED240400000006036A357C000000020FCD0404000000040FAA357C00000004059D1404000000000FBA357C00000004048D2404000000000AFA257C0000000C0F9D2404000000020E8A257C000000000B0D44040000000C0E4A457C0000000E079D3404000000020A8A557C0000000C0E5D340400000004073A557C0FFFFFFBFC0D44040000000602CA457C0000000A0DDD44040000000E0E2A357C00000004001D640400000004006A557C000000000D2D54040FFFFFF5F72A557C0000000E0E8D64040000000E075A657C0FFFFFF9F32D54040000000606EA857C0000000008AD7404000000080E1A857C0FFFFFFFFC5D540400000008025AA57C000000060F5D44040000000E0D7AA57C00000008041D54040000000C0BFAA57C000000060F2D54040000000A03CA957C000000040BDD6404000000060F3A957C0000000009DD8404000000000C8AA57C00000002028D940400000002039AC57C0000000605BD840400000004077AF57C000000020C3D940400000006049B057C00000004098DA4040FFFFFFBF7EAF57C0000000C009DC404000000060D0B057C0FFFFFFFFBFDB404000000040FCAF57C0000000604CDE4040000000401FB257C0FFFFFFBFE8DD4040FFFFFFBF0CB257C0000000A002DF404000000040E8B057C0000000205DE040400000000025B257C0000000E06AE04040FFFFFFFF67B357C0000000A0AADE4040000000206DB457C0FFFFFF7FECDF404000000080E7B657C0000000C0E9DF4040000000C06CB857C00000000032E34040000000007DBA57C00000008011E54040FFFFFF9F25BA57C000000060D8E6404000000060C4BA57C00000002079E84040000000802FBC57C0000000606CE840400000002027BC57C000000020A0EB4040FFFFFF1F6FBD57C0000000C08DEC404000000040FFBD57C000000020E0EE40400000006050BF57C00000006097ED404000000040D1C057C0000000605AEF40400000002064C257C0000000E0E7EE404000000040BEC257C0FFFFFF7F35F14040000000800AC457C0000000E0C6F240400000000010C457C00000008075F54040000000C059C557C000000020B9F14040000000E0BDC557C000000020CEF24040000000E043C557C0FFFFFFDF8FF54040000000601AC657C000000080FBF5404000000060A1C757C0000000A0C5F44040000000801BC857C00000000065F54040000000A030C857C0000000606EF84040000000007EC957C000000020C6F8404000000080FACE57C0000000A080FB4040FFFFFF1F15D057C000000060DDF740400000004010D057C0000000C0D7F34040FFFFFF1FDFD057C000000020EBF2404000000020C0D157C0000000007FF54040000000E054D257C00000000086F140400000004053D357C0000000E07CF14040000000C084D557C0000000A0D4F24040FFFFFF5F1FD557C0000000407AEF404000000020E7DC57C000000000D1EE4040000000C0F5DD57C0000000A076F1404000000040EDDF57C000000020DCF0404000000020D3E057C000000000E9F2404000000080D1E257C00000000060F14040000000200AE357C00000000053F2404000000000B6E157C0000000A0EBF24040000000C040E157C0000000E00CF44040000000E0F6E257C000000060B7F340400000008004E457C000000040D1F74040000000E0C9E657C0FFFFFF1FE7F840400000006059E757C000000080E5F74040000000203BE757C0FFFFFF5FCAF54040000000208BE857C000000000C6F5404000000000C8EC57C0000000A089F24040000000A0CCEF57C000000080A2F3404000000040AFF057C0000000405CF24040FFFFFF7FE9F057C00000004083F04040000000602FF157C0000000C0FAEC404000000020E9F257C0000000A0ADEE4040000000C0DCF457C0FFFFFF3FE8EB4040000000002EF657C000000020A7EB4040FFFFFF7FB7FB57C0000000E0FCF14040000000405BFC57C0000000A0EAF14040FFFFFF5F5CFD57C0000000A0B9EE4040000000A08DFE57C000000060D1ED404000000020A1FF57C0000000600CF04040FFFFFFDF2A0058C000000040CBEF4040000000601D0058C0FFFFFF7FB1ED404000000080E60058C0000000000FEC404000000040B60158C0FFFFFF1F92ED4040FFFFFFFF110358C000000000AFEB4040FFFFFF7FDB0558C0000000201BEC404000000020010758C00000002025EA4040000000A0880958C0000000A0F4EA404000000040D40A58C0000000201CEA404000000060B80B58C0000000E06BE8404000000000910B58C0000000C07AE74040000000A0E30958C00000000077E94040000000000D0958C00000004000E9404000000000530A58C0000000602CE64040000000E0CD0A58C0000000407AE2404000000040F80B58C00000004019E14040000000609A0D58C000000040DBE04040FFFFFFFFCB1158C000000060FEE24040000000208A1258C00000000087E1404000000020401358C00000000066DB4040000000E03D1458C0000000A0D4D94040000000E03E1658C0000000C04EDA404000000080BB1758C000000040C5DE4040FFFFFF7FD81A58C0000000E0E7E4404000000020311F58C0000000C099E34040000000400C2058C000000020E0E4404000000040AD2058C00000006068E8404000000000FA2358C000000060A7E94040FFFFFFFF792658C000000000E6EB4040000000804E2758C00000008073EE404000000040682558C0000000E0B4F24040000000A0A32A58C000000000EFF44040000000805F2B58C0000000E0C0F3404000000060602C58C00000002088EC4040000000208C2D58C0FFFFFF3FBCEA4040000000C0EC2F58C00000006076EA4040000000C00B3358C0000000805AEF4040000000801A3458C00000002096EF404000000040043658C000000000D4ED4040000000E01A3758C0000000804BEE4040FFFFFF7F403858C00000000027F14040FFFFFF9F803858C00000000059F64040FFFFFF3F803958C0000000809AF94040FFFFFFFF7D3B58C0000000601BFB4040000000C0EA3B58C00000002053F94040FFFFFFBFF63D58C000000020FAF7404000000020393F58C0000000A0DBF84040000000A0363F58C0000000A02EF04040000000E05F4058C0FFFFFF9FDDEC404000000060A34158C00000008097EB4040000000A0894458C000000040A9ED404000000060424558C0000000E0F0EC4040FFFFFFFF014558C0000000603DEB4040000000A0334358C0000000C066E94040FFFFFF5F9C4558C0000000805EE7404000000080574558C0FFFFFF5F07DF4040000000C0CA4558C000000060A7DD404000000060654758C000000060EBDC404000000020C24958C00000000045DD4040000000401B4C58C0000000C05AE0404000000020554D58C000000040EAE84040000000207B4C58C00000004007EB4040FFFFFF3FCA4A58C00000008084EC4040000000C0814A58C0000000A07BEE4040FFFFFF5F044C58C00000002019F3404000000080864D58C0000000A0EDF3404000000080BF4F58C00000006076F24040000000400B5058C000000080BDEF4040000000E0E35058C0000000E0EAED4040000000006D5158C000000080B0EF4040000000001A5458C0000000E0AAF2404000000040285458C00000000069EF404000000020E05558C00000004053EE4040000000A0405758C0000000005FEA4040000000603F5A58C0000000000DE94040000000A0F95C58C00000000009EB404000000080405D58C0000000A0F9F14040000000E09D5D58C00000004081F34040000000E08F5E58C0000000C02FF44040FFFFFF3F2A6158C0000000C058F54040FFFFFF5F7E6358C000000020B3F34040000000C0D76458C0FFFFFF1F86F3404000000020E96558C0000000407DF54040FFFFFF5F696658C0FFFFFF7F16FC4040000000C0F26A58C0000000E08AFE4040000000A0126D58C0FFFFFF9F5BFC404000000040A86E58C0000000C03AF8404000000040687058C0000000004FF74040000000C0927258C000000080FAF1404000000020907658C000000080B4ED404000000000AA7758C00000006074ED4040000000202E7A58C000000000E0EF4040000000601A7D58C0000000E015F14040000000007D7E58C00000004085F34040000000E0787E58C0FFFFFF1FBEF4404000000000D87C58C0000000C05CF74040000000C0A17D58C0000000606DF9404000000000A87C58C000000020D9FA404000000080D07C58C0000000004FFC404000000040E47E58C0000000202A004140000000E0808158C0FFFFFF7F55FE4040000000408E8358C0000000C0B1FE4040FFFFFF5F848558C000000020AE00414000000080158758C0000000E0EF084140FFFFFF1F068658C00000002039114140000000005A8758C0000000001213414000000060C28858C000000020B8114140000000E00F8B58C000000060C40E414000000060BA9158C000000040BA0F4140000000E0829458C000000080D8114140000000006D9658C00000000031124140000000A0979858C0000000E0D10E414000000060059958C0000000602A0B4140000000800E9A58C0FFFFFFDF8D0A414000000040F79A58C0000000206D08414000000020AF9C58C0FFFFFFBFF506414000000020F89F58C0FFFFFF3F8008414000000060AFA358C0000000A07B0D4140000000A0E2A458C0000000C02A12414000000020DDA658C000000000611341400000004010A858C0000000A047144140000000A059AA58C0FFFFFF1FD212414000000060A9AB58C000000020331341400000008023AD58C000000040BB104140FFFFFF7FD3B158C000000020E4104140FFFFFF7FE8B358C000000000AE124140000000C00BB958C0000000C095144140000000E0F5BC58C000000080E7184140000000A0C1BF58C0000000C0D01A41400000000041C258C000000040761941400000000005C558C000000080AB1A4140FFFFFF3F30C858C0000000C0C91941400000002046CB58C0000000C03A1B4140000000E030CC58C000000060A31C41400000002018CD58C000000000B9204140FFFFFF3F90CC58C0FFFFFF3F0E274140000000C026CD58C0000000C07E2A41400000004043D058C0000000A0212F41400000006019D158C000000000FB324140000000E0B0D458C0000000A0D3344140000000204FD758C000000000A03941400000008023D958C000000040E9364140000000E039D958C000000080C8324140000000C03CDA58C0000000E03E2F4140000000600EDC58C0000000A0AE2E414000000020AFDE58C000000040173141400000000023E058C0FFFFFF7FB83341400000008072E358C0000000A02435414000000080FBE458C000000040573441400000004074E558C00000000043314140FFFFFF1F7EE658C0000000E02C2F414000000080D5EB58C00000002050304140000000A0C5F158C0FFFFFFBFD4384140FFFFFF9F1DF558C0000000403A404140000000A013F758C00000006062424140000000C05BF858C0000000A02A46414000000060A4FB58C000000040204A4140000000A076FC58C0FFFFFF5F2F4A4140FFFFFFDF36FE58C000000020EB47414000000000C0FF58C000000020FA474140000000A0EDFF58C0000000C0A35F4140000000A0BFFF58C0000000C0F7834140FFFFFFDFD1FF58C0000000C05197414000000080A3FF58C0FFFFFF5F58B6414000000000000059C00000002035CF4140000000C0DAFF58C00000002020F1414000000060D1FF58C0000000C05D074240FFFFFFBF120059C0FFFFFFDF0A3F424000000080700059C000000060373F4240FFFFFF9F292359C000000040A63E4240000000003E3D59C000000080AB3E424000000020BD4559C000000080773E4240FFFFFF3FB36759C000000000FA3E424000000080308259C000000020193F424000000000938A59C000000020BF3E424000000080CDBF59C000000020053F4240FFFFFF7FD2BF59C000000000CF7F4240000000405A8259C0000000A0967E4240000000608A8159C0000000C0927E424000000040686359C0000000C0927F4240FFFFFFDF8D4459C000000020AC7F424000000020CF3C59C0000000C0917F424000000060902859C000000020B87F4240FFFFFF5FA40559C000000020B27F424000000020EDFF58C0000000E0687F4240000000E0D4E258C0FFFFFF5F6A7F4240FFFFFF1FF9DB58C0000000C04C7F4240000000E0F1BF58C0000000E0BF7F4240000000208DA258C0000000E0C97F4240000000E0319658C000000060E07F424000000000AB8658C0000000A0D37F424000000060737358C000000040D07F4240000000E0C35D58C0000000608B7F4240FFFFFFBFCA4858C0000000C0F87F4240FFFFFF9FE52F58C00000006004804240000000A0352158C0000000A011804240000000E05E0058C0FFFFFF1FC77F4240000000004BFD57C000000060008042400000008045F257C0000000A0BF7F424000000020A6E157C0000000C01E8042400000000002DA57C0000000A011804240000000A096C457C0000000202E804240000000A014C257C00000004017804240FFFFFF9FB0A757C0000000209D7F424000000000C6A757C0000000A0BB61424000000000BCA757C000000000D2554240FFFFFF7F7DA757C0000000A0A23E4240000000E0DCA657C000000080463D4240FFFFFF9F62A357C0000000800C154240FFFFFF5FB3A257C0FFFFFF1FAA0D4240FFFFFFFF159F57C0000000204FE14140FFFFFF1FF89D57C0000000600CD24140000000E0699B57C00000002042B34140000000401A9C57C000000020EB764140 Oklahoma 40 0.39496527777777777778 0.35555555555555555556 0.34165720771850170261 0.31952662721893491124 0.35463258785942492013 0.37058823529411764706 0.41274238227146814404 0.36981566820276497696 0.39620653319283456270 0.43522012578616352201 0.38820912124582869855 0.36416747809152872444 0.37113402061855670103 0.40179717586649550706 0.49089767733835530446 0.59224515322076297686 0.59002433090024330900 0.54183266932270916335 0.58079096045197740113 0.62940528634361233480 0.64598337950138504155 0.55132530120481927711 0.58344640434192672999 0.58448060075093867334 0.60277891295463833265 0.60372670807453416149 0.59992085476850019786 0.57948344852673699527 0.58967971530249110320 0.65690223361406078360 0.65572033898305084746 0.65481886534518113465 0.64003944773175542406 0.62625628140703517588 0.63230769230769230769 0.64569244339900029403 0.65894501814122243930 0.64971605575632421270 0.64410011918951132300 0.66351564258383974792 0.65978956055291933155 0.68310412573673870334 0.70018867924528301887 0.70563454449710373881 0.72488383271911552636 0.73183619550858652576 0.75631993369249896395 0.75764109067850348763 0.75596877869605142332 0.75531697341513292434 0.77340260687266429678 0.77015837285955462658 0.79358095924990984493 0.79292759847010668993 0.74213557820115197164 0.72167235494880546075 0.70697649629590150829 0.66314003393552250724 0.61493491664763644668 0.59386845503394843171 0.59173547311702911191 0.60644823459006582885 0.62245467743736738265 0.61204819277108433735 0.61093845010472265387 0.61345473601467312983 0.60706795630262622468 0.60202557361376673040 0.59293179008877401955 0.59273203033215849621 0.58404580152671755725 0.57866633858267716535 0.59270559852408997375 0.59634468480045691440 0.60660880230199537714 0.61134439778710924482 0.61932712347411849772 0.62098840279271703205 0.60784350083441498238 0.63046508215962441315 0.63918385922330097087 +10 0106000020E61000000100000001030000000100000070010000FFFFFF9F9E415DC000000060EED64540000000A075415DC0FFFFFF1F7DE04540000000E04F425DC0000000E06EE6454000000060B4415DC0000000A074EA4540FFFFFFDF9B405DC0FFFFFF5F81EB4540FFFFFF7FF9405DC00000000032ED4540FFFFFF9F063F5DC0FFFFFFFF02EE4540000000C0893E5DC0000000A0D1EF4540FFFFFF9F893E5DC000000080C8F34540FFFFFFBF5B3D5DC000000060DFF64540000000C0E23D5DC0000000C04DFB4540FFFFFFBFAF3B5DC000000020D5014640FFFFFFDF733E5DC00000002079094640000000C0983D5DC0000000A0920B4640FFFFFF7F893C5DC0FFFFFFFFEB0B4640FFFFFF3FAE395DC000000020BE124640000000205F3A5DC0000000A0B5164640000000A0C63E5DC0000000A05619464000000080683E5DC0FFFFFF7FD61C4640FFFFFF1F783F5DC000000080A31F4640FFFFFFDFE0415DC000000000EE1F4640FFFFFFFF43435DC000000060A71D46400000000025455DC0FFFFFF1F3A1F4640000000205F465DC00000006033224640FFFFFFDF25475DC0000000C08C224640000000001B495DC00000008018204640000000A0DC4A5DC00000000071204640000000A09A4D5DC00000008075244640FFFFFF3FDA4D5DC0000000008026464000000080D64C5DC000000080762B464000000020194F5DC0000000C0EE314640FFFFFF5FD64D5DC0000000C0C7364640FFFFFF1F4C4E5DC0000000A08E3C4640FFFFFF1FFD4C5DC000000000323E4640010000A0ED4B5DC0000000A086414640FFFFFFBF39495DC00000006073444640000000C025495DC0000000809A474640FFFFFF9F49485DC0000000404C494640000000A003455DC000000000405846400000004031445DC0000000604D594640FFFFFFDF77425DC0FFFFFFDFE65F4640000000E0D43C5DC00000000059634640000000E0263A5DC0000000A01E6A4640000000205E395DC000000060A16C4640FFFFFFDF6D375DC000000080326F4640010000E066355DC000000040CB754640000000202E365DC0000000803C7A4640FFFFFF5F25355DC000000040837C4640FFFFFFFF37365DC000000000667C4640FFFFFF9FB6365DC0FFFFFF7F737D46400000000037365DC00000006005804640FFFFFF1FA0365DC0000000402F824640000000C09A335DC0FFFFFF5F628646400000008051325DC000000040B689464000000020BC315DC0000000C0BF8C4640FFFFFF7FA8305DC0000000409F8D4640FFFFFF1F142F5DC0000000409791464000000080052C5DC0000000A098A1464000000060F62A5DC0000000A0F2AA4640000000A025245DC0FFFFFF9FE0BA4640000000006D235DC0000000A033BF4640FFFFFF9F901E5DC00000006078C84640FFFFFF5F0B1E5DC0000000A09DCD4640FFFFFF5FE4205DC0000000E011D54640FFFFFF3FBF215DC000000000FDDA464000000060D1235DC000000080AFDF4640FFFFFF9FD1295DC0000000A0EFE34640000000E0F02C5DC0FFFFFF7FDEE846400000006074315DC000000000F2E84640FFFFFFFF93325DC00000006049EC464001000060C0365DC000000020ADF34640FFFFFFFF6B395DC0000000A085FD464000000000C33A5DC00000000066FF4640FFFFFF3F3B3D5DC0000000806C084740010000601E3F5DC0FFFFFF9F100A4740000000607B3D5DC000000080770C4740FFFFFF9F6B3B5DC0FFFFFF7F32154740FFFFFF3FDB3D5DC0000000604D19474000000000313E5DC000000040ED1F4740000000C00A405DC000000080BA264740FFFFFF1FBA415DC0FFFFFF1FF32A4740000000600B445DC000000000A62C4740FFFFFF5FC8425DC0FFFFFF9FC03147400000008067425DC0000000E0CB364740FFFFFF9F9E425DC0FFFFFF1FB3444740FFFFFFBF8E425DC000000040498F4740FFFFFFFFA5425DC0000000E019A14740000000209C425DC00000000047AE4740FFFFFFFF54425DC00000000050FC4740FFFFFFDF6B425DC0000000A0EC054840FFFFFF7FCB415DC000000080486B4840FFFFFF7FFB415DC00000000000804840FFFFFFFFFF035DC0000000E0FF7F48400000004090035DC0FFFFFF3FD63F4840000000E07C035DC0000000C0B11A4840FFFFFFDF5B035DC000000060F6FC4740000000408E015DC0000000A085FB474000000000DEFF5CC0000000806DF6474000000020F5FB5CC00000006000EF474000000060C9F95CC0000000C0AAEB4740FFFFFF7F9AF75CC0FFFFFF5FECE94740000000E04AF65CC0000000A012E74740000000C07CF55CC000000040D0E04740FFFFFFBF87F25CC00000004059DF47400000002095F15CC0000000E0DADA474000000000E4EE5CC0000000400AD94740000000A0EAEE5CC0000000E0E9D14740FFFFFF5FA0EC5CC000000000DECE4740FFFFFFDF3AEC5CC0000000009FCB4740000000407AEF5CC00000004052C44740FFFFFF7FD5EC5CC0000000E0AEC24740FFFFFF1F46EC5CC000000080ABBE4740FFFFFFDFE7E85CC0000000C0D6BC4740000000C019E95CC0000000409BBA474000000020EDE95CC0FFFFFFFF80B94740000000C0D0EE5CC00000000002B9474001000080F1EF5CC0FFFFFF7F8EB7474000000020F5EF5CC0FFFFFFFF15B64740FFFFFF5F97EA5CC0000000401AB34740000000A0D1E85CC0FFFFFF9FA7B04740000000A04EE65CC0000000A05EAF4740000000807DE35CC000000000D7AA4740FFFFFFBF69E15CC0FFFFFF1F62A54740FFFFFF7F0FE05CC0FFFFFF5F0FA44740000000003EDB5CC00000002021A34740FFFFFFFF0FDA5CC0000000E0BFA1474000000080EED55CC000000000A7A04740FFFFFF7FC0D45CC000000080639F474000000080E5D25CC000000000FE964740000000604BCC5CC0000000A0E28F4740FFFFFF3FFACA5CC0000000C07F8C47400000002074C95CC0000000C0C08B4740000000409CC85CC00000000025884740010000C024C55CC0FFFFFFBF67834740000000C07FC35CC0000000A0997C4740FFFFFFDF53C25CC000000040467B4740000000000AC05CC0FFFFFFDFBC7A474001000060AEBD5CC000000000707647400000002015BB5CC0000000A0207447400100008022BC5CC0000000400B724740FFFFFFFFA2BC5CC0000000A01F6D474000000000B0B95CC00000008056664740FFFFFFBF66B75CC00000004008664740FFFFFFDFBFB55CC0FFFFFF7F47634740010000C0C1B25CC00000006020624740FFFFFF5FB8B15CC000000000BE604740000000C00BB25CC0000000E0FF59474000000060D1AF5CC0000000C0FC58474000000080A3AC5CC000000080EE5D47400000006011AB5CC0FFFFFFFF0D5E4740FFFFFF9FFFA75CC0000000C0F5574740000000C038A95CC000000060E3554740000000E033A95CC000000080985447400100004008A75CC000000040875047400000004086A25CC000000000A951474000000020F69E5CC0000000E0D34F4740000000A0329C5CC000000060A95247400000004084985CC0000000C0B1544740000000005C955CC000000040C0534740000000A0B8945CC0000000C0BB4F4740FFFFFF7FE9955CC0FFFFFF3F424B4740FFFFFFDF58965CC0000000603C4247400000008002975CC000000040CB404740000000C0399A5CC0000000C0643E4740FFFFFF7F84985CC000000020D1364740FFFFFF5F59995CC0000000E026334740000000C02A9A5CC0000000804D32474000000040939B5CC0000000007424474000000060439E5CC0000000A063204740000000A0339E5CC0FFFFFFDF361F4740FFFFFF5F129C5CC0FFFFFFDF331C474000000020289C5CC000000060A315474000000060D49D5CC000000040E2134740000000408EA05CC0000000E0281447400000002027A15CC0000000206D114740000000605CA05CC0FFFFFF5FE00E4740000000007E9E5CC000000080C00D4740FFFFFF5F209D5CC000000020890A4740FFFFFFBFBF9D5CC0000000C083064740000000C0939F5CC0000000C001034740000000804F9E5CC0000000A042014740FFFFFF1FF29E5CC0FFFFFF9FB4FE464000000060569A5CC0000000406CFC4640FFFFFF1FE1995CC0000000A01EFA4640000000206D9B5CC0000000A0F5F54640FFFFFF1F689A5CC0000000E092F446400000002015995CC0000000E07BEF4640FFFFFFBF059A5CC0000000405BEC4640000000C04E9C5CC00000006025ED4640FFFFFF9F439E5CC00000006076EB464000000020E39F5CC0000000C0DFEB4640FFFFFF7F0DA15CC000000000B3E74640000000E001A45CC00000000099E1464000000040A0A25CC0000000A07FDF4640FFFFFFDF2DA25CC0000000E08DDC4640000000E0C89F5CC0000000E0E4D8464000000040A0A05CC0000000404AD64640000000600DA05CC00000006084D346400000008014A45CC0000000E0EACF4640FFFFFFDF8FA25CC00000008059CC4640FFFFFFFFD6A35CC0000000E03FC64640000000A0B1A15CC0000000A075C7464000000000B99F5CC000000060FBC54640000000E08D9D5CC0000000A022C6464000000040B09B5CC0000000408CC34640000000A0C29A5CC000000040E2BF4640000000E0B6975CC0000000802FBE4640FFFFFF5F5B965CC0000000E052BB464000000040D5945CC0000000A08FBA4640FFFFFF7FB9905CC00000006033BE464000000040CE8F5CC00000004063C04640000000E06D8F5CC0FFFFFF1F87C44640000000C0688C5CC00000008095C3464000000000FE8A5CC000000000A2C546400000008064885CC0000000A075C64640000000A080875CC0000000601DC946400000006040855CC00000002011CB4640000000008F835CC00000006007D046400000008018815CC00000008007D2464000000020A5805CC00000002088D34640000000C036815CC0FFFFFF1F13D64640FFFFFF3F8C805CC0FFFFFF5FDCD7464000000060187E5CC00000004046D94640000000A0A57D5CC000000080F8D64640FFFFFF1F407B5CC0000000E0ECD54640FFFFFF3F0C7B5CC0000000C0DDD34640000000C0AE795CC0FFFFFF1F94D14640FFFFFFDFC0795CC00000008089CE4640000000C078765CC0000000C008CE4640FFFFFFBF94745CC000000040E4CC4640000000E061735CC0FFFFFF3FBACA4640000000C04C755CC0FFFFFF1FEBC14640FFFFFFFFEB715CC0FFFFFF3F2BC24640FFFFFF9F5E715CC0FFFFFFBFE9C04640000000402B715CC00000002028BD46400000008039725CC0FFFFFF1F0DB94640000000408B715CC00000002083B44640000000E0626F5CC0FFFFFF9FF4B04640FFFFFF5F3E6F5CC0000000802AA9464000000020056C5CC00000002091A34640000000401C6C5CC00000000095A14640000000C042695CC0000000A07A9A46400000000067665CC0000000402D97464000000020F7655CC0000000402D934640FFFFFF3F86645CC0000000403B91464000000000B4645CC0000000A0138F4640000000E049635CC0000000A0C68D46400000000099605CC000000000D08D4640FFFFFF1F43615CC000000080838A4640FFFFFF3F505F5CC00000002020894640FFFFFF1F115F5CC0000000807987464000000060175D5CC0FFFFFFDF8E854640000000604F5D5CC0000000E085834640000000602B5C5CC000000040C77F464000000020AA5C5CC0000000608C794640000000209A5D5CC0000000C06D78464000000040225F5CC0000000604678464000000020B35F5CC0000000A022774640000000E0755C5CC0000000E0F76C464000000080E95A5CC000000020B96A46400000008058565CC0000000E060674640000000A0BE555CC0000000C0B86346400000000057545CC0FFFFFFFFE0634640FFFFFFFF66505CC000000080C067464000000080534F5CC0000000E0EC67464000000060CB485CC00000006079614640000000C019485CC000000000655E46400000004074465CC0000000608D5B46400000004047465CC0000000601659464000000060A0445CC0000000407A564640000000806B435CC0FFFFFF5FEB4F46400000008042455CC000000000984A4640000000E07B425CC0FFFFFF5F37474640000000C05D425CC0000000C03A444640FFFFFF7F70405CC0FFFFFF7F5E414640000000E03E415CC0000000A0AD3D4640000000405F405CC000000020F2394640FFFFFFFFB6405CC0000000E00938464000000080003F5CC0000000A0C237464000000000FF3B5CC0000000C021344640000000E0BB385CC0000000E04B32464000000020AD375CC0000000605E2F4640000000E0FA355CC000000000472D46400000008042345CC000000020A12E46400000000058345CC0000000608C324640000000C0DA345CC000000000E9354640000000A0E7325CC000000000A33A4640FFFFFF7FDB315CC0FFFFFF5FAC3C464000000060E72E5CC000000020013E4640000000C0A92D5CC0000000809E3F4640FFFFFF1FC0295CC0000000E08D3D4640000000A076225CC0000000A0213D46400000008010205CC000000080463B4640FFFFFFBF4A1D5CC0000000C0053C464000000000E01A5CC00000002085394640000000E078175CC0000000A08439464000000060BE155CC000000080A63F464000000000DE155CC0FFFFFF5F39434640FFFFFF3F04125CC00000006059454640000000C05F105CC000000000B0474640FFFFFF3FB10E5CC000000040A0474640FFFFFF3FE20D5CC0FFFFFF3FF0444640000000A0B90C5CC0FFFFFF5F0944464000000040E5075CC0000000A0A04346400000004057065CC00000004058424640FFFFFF3FBF035CC000000060AC43464000000040AE015CC000000060EF4246400000008075015CC0FFFFFF1F7F4446400000004087FE5BC000000040CF4346400000000022FC5BC00000004060464640FFFFFFDFC9F75BC00000008036474640FFFFFF7FA6F35BC0000000408540464000000000ADF25BC0000000C05F4246400000002008F15BC0000000A06B42464000000040D6ED5BC00000000055444640000000C0C7EB5BC0000000C08146464000000060AFE65BC00000006083454640FFFFFF7F40E45BC000000000C7464640FFFFFF1F53DF5BC000000000AF43464000000080D5DE5BC000000000A3444640000000A058DD5BC0FFFFFF3FDD444640FFFFFFFF91DD5BC00000002067464640000000C07EDF5BC0000000009046464000000000E1E05BC000000080F04B4640000000A00FE05BC0000000C0DA4E46400000000071E05BC000000060A2514640FFFFFF3F0ADE5BC00000006005524640FFFFFF3F47DD5BC00000008089534640000000006FDD5BC0000000E0C555464000000080B8DE5BC0000000E0825846400000006060DE5BC0FFFFFFFFE25946400000008057DC5BC0FFFFFFFF4B5B4640000000203CD95BC0000000C0BE5A46400000004096D85BC0000000406F5E464000000000C7D75BC000000080615F46400000002059D65BC0FFFFFFDFF55C46400000004061D45BC0000000202D5D4640FFFFFFDF23D45BC000000040465A464000000060DFD25BC0FFFFFFFF6C574640000000403ED15BC0000000A041564640000000C045D15BC00000008036524640000000C048CE5BC000000000415046400000006004CE5BC0000000801C4F4640000000E0F0CE5BC0FFFFFF3F234D4640000000A0FFCD5BC0FFFFFF1F604946400000002064CB5BC0000000804F48464000000080D8CA5BC000000020CB454640FFFFFF9F8CC85BC0000000C0944346400000008033C85BC0FFFFFF3F1B404640000000C001C65BC0000000C03B3E46400000002040C35BC0FFFFFF3F983C4640000000202DC35BC0000000E0C6FD4540000000A0F1C25BC000000060FFC14540000000A0FDC25BC0000000C074A44540000000C019C35BC0FFFFFF1F8E824540FFFFFF1FF2C25BC0000000E06C404540000000A011C35BC0000000A085FF4440000000C09ADF5BC0000000E007004540000000E061065CC0000000E04D004540000000605D095CC000000080E3FF4440000000C0473F5CC000000040280045400000004072825CC0000000C06BFF4440FFFFFFFF30915CC0000000407DFF4440FFFFFFDF88C15CC0FFFFFF7F90FF44400100006095FC5CC00000004052FF4440FFFFFF5F723F5DC00000000059FF4440FFFFFF5F25415DC00000000059FF4440FFFFFF9F9E415DC000000060EED64540 Idaho 16 0.44010416666666666667 0.48599033816425120773 0.42451759364358683314 0.40532544378698224852 0.36261980830670926518 0.59264705882352941176 0.55263157894736842105 0.54723502304147465438 0.44573234984193888303 0.53584905660377358491 0.48609566184649610679 0.45082765335929892892 0.51202749140893470790 0.58664955070603337612 0.64344005021971123666 0.68480300187617260788 0.69038929440389294404 0.68468981217985202049 0.72203389830508474576 0.74063876651982378855 0.70747922437673130194 0.64048192771084337349 0.67706919945725915875 0.68585732165206508135 0.63342868818961994279 0.64554865424430641822 0.63157894736842105263 0.62859221535103674063 0.63451957295373665480 0.66532405712193335774 0.66772598870056497175 0.64866712235133287765 0.64924391847468770546 0.65703517587939698492 0.66707692307692307692 0.66157012643340194061 0.71281049399944180854 0.66881775942178626742 0.65673420738974970203 0.65496286293045239703 0.67443779657520115535 0.69901768172888015717 0.70962264150943396226 0.72845357205546779006 0.75452651818618811088 0.78996036988110964333 0.76958143389970990468 0.77564996829422954978 0.74724517906336088154 0.74836400817995910020 0.71953331510345456203 0.70222686711150414020 0.67832672196177425171 0.64557471650003355029 0.65288942338122665992 0.62963594994311717861 0.62074295155358951127 0.59726519612735801976 0.57597625028545329984 0.56433540217436580997 0.57382641392409970152 0.59343207660083782166 0.60287384134311134274 0.60192072638379605378 0.61678264982095804338 0.61725402856019913533 0.61445519141077409459 0.60806046845124282983 0.58517810989998876278 0.58536259745807967532 0.58104325699745547074 0.59498031496062992126 0.59424300480143806618 0.60648247304918969087 0.61116090381621774612 0.61867626474705058988 0.61679639317766152014 0.60536248606575010267 0.59647691451882069349 0.59275968309859154930 0.58758722694174757282 +45 0106000020E610000003000000010300000001000000F4000000FFFFFFDFA5995EC0FFFFFFBFD91C4840000000A08A9D5EC0000000E0401D484000000080009D5EC00000006078104840000000800B975EC000000040B707484000000000D9A05EC0000000002C114840FFFFFF7F9CA25EC000000040F61A4840000000C080A05EC0FFFFFF1F82204840FFFFFFFFCC995EC0FFFFFF3F961F4840000000A021985EC0000000601B254840000000A009A45EC0FFFFFFBF0B354840FFFFFF9F9BAA5EC000000080DF34484000000020AEAC5EC0000000E04B3F484000000060D7A65EC0000000806E424840000000A067A15EC0000000A0B23A4840000000603E9E5EC0000000E02E3B4840000000E041A05EC000000080A1474840FFFFFFBF689B5EC0000000A0BE4C4840FFFFFF1F239F5EC000000040C2514840000000209EA15EC0000000601F5B4840FFFFFF1FFFA05EC00000002009614840000000208DAC5EC0000000C0CE6648400000006030B05EC00000000080744840000000608DB45EC0000000E0B6794840FFFFFF9F87AF5EC0000000605D7A4840FFFFFF9FE2B05EC000000020FD7F4840010000E0C5365EC000000020FF7F4840000000C0EAB55DC0000000000080484000000040C08C5DC0FFFFFF1F0080484000000000005C5DC0000000E0FF7F4840FFFFFF7FFB415DC00000000000804840FFFFFF7FCB415DC000000080486B4840FFFFFFDF6B425DC0000000A0EC054840FFFFFFFF54425DC00000000050FC4740000000209C425DC00000000047AE4740FFFFFFFFA5425DC0000000E019A14740FFFFFFBF8E425DC000000040498F4740FFFFFF9F9E425DC0FFFFFF1FB34447400000008067425DC0000000E0CB364740FFFFFF5FC8425DC0FFFFFF9FC0314740000000600B445DC000000000A62C4740FFFFFF1FBA415DC0FFFFFF1FF32A4740000000C00A405DC000000080BA26474000000000313E5DC000000040ED1F4740FFFFFF3FDB3D5DC0000000604D194740FFFFFF9F6B3B5DC0FFFFFF7F32154740000000607B3D5DC000000080770C4740010000601E3F5DC0FFFFFF9F100A4740FFFFFF3F3B3D5DC0000000806C08474000000000C33A5DC00000000066FF464000000000C35E5DC0000000E0FEFF46400100000084665DC0000000200D004740FFFFFFFFD27E5DC0000000A00000474000000060747F5DC0000000403A00474000000000C9BE5DC0FFFFFFDFE5FF4640000000A0FDC15DC0000000A0B3FB464000000080E7C85DC00000006082F64640000000205ECB5DC00000006014F64640000000004ED35DC00000004066F746400000004036D85DC00000000079F54640FFFFFFBF03DC5DC0000000800BF54640000000A0B5E05DC0000000C01DF3464000000060A4E55DC000000040ECF44640FFFFFF1FBEE75DC0000000A024F346400000000059EB5DC0FFFFFFBF24ED46400000004046F55DC0000000A0BEEB46400000000097F75DC0FFFFFFDF79EA46400100002090FF5DC000000040D8E74640FFFFFFDF51045EC000000080E2E34640FFFFFF7FE7095EC0FFFFFFDF75E14640FFFFFFDF330D5EC0000000C026DC46400000002014125EC0000000E0BDDB4640000000604D1C5EC0000000403FD8464000000020DF1F5EC0000000600FD946400000002069245EC000000060DBDE464000000080D8275EC0000000A033DF4640FFFFFF1F102A5EC000000040CBDD464000000060882C5EC000000000F7DA4640FFFFFF3F0E375EC0000000E029D5464000000060083A5EC0FFFFFF5F5CD1464000000020A23C5EC0000000A042D3464000000040E83D5EC00000008099D24640FFFFFF3F11425EC00000008095D3464000000060A1445EC0FFFFFF3FC9D2464000000000F0475EC000000040B9CD4640000000A0144B5EC0000000E0E2CC4640FFFFFF3F374C5EC0000000E083CE4640FFFFFF9FEF4C5EC00000002027D44640FFFFFF3FA34D5EC00000000039D54640000000009D515EC000000000D9D64640FFFFFF1F67545EC0000000C030D94640000000E076575EC00000008094D9464000000020EF5A5EC0000000E06AD84640000000603F5C5EC0000000E0F9D84640000000A0C8615EC00000000020DC464000000080226D5EC0000000A02FD84640FFFFFFFF7A705EC0FFFFFFDF4DD84640FFFFFF9FD4735EC000000040B5D94640FFFFFF1FC6785EC000000080A8D64640FFFFFF7F3D7B5EC00000004033D24640FFFFFF7F2C7E5EC00000006066D14640FFFFFF9FEC7F5EC0000000201ACF4640000000802C855EC0000000E09ACB4640FFFFFF1F998F5EC0000000C02DC64640FFFFFF1F53935EC00000004089C5464000000080BC965EC0000000807DC84640FFFFFF9FE69B5EC0000000E04FC84640000000401CA45EC00000004028CC4640FFFFFF9F99A95EC0000000E0B1CD4640010000C07CAC5EC000000060CBD04640000000E098B05EC0000000C024D34640000000A05DB15EC0000000202ADD4640FFFFFF3FD6B05EC0FFFFFF9F5FE14640010000E05AB25EC00000000077E64640000000A021B25EC0000000E0E0EC4640000000601AB25EC0000000401CEF46400000004085B35EC000000000BEF34640000000209EB35EC0000000C0D6F84640010000E0F2B75EC0FFFFFF1F80034740000000A081B95EC0FFFFFFDF2C0A4740FFFFFFBF44BE5EC0000000C0290E4740000000E028C35EC0000000A0F4134740FFFFFF3F82C75EC000000020F9164740FFFFFF9F32CB5EC0FFFFFF3F85174740FFFFFF5F84CD5EC000000040C8154740FFFFFF1FD8CF5EC0000000C074124740010000406CD35EC0FFFFFF3F8C124740000000C00CDE5EC00000008039234740FFFFFFDF9AE75EC00000008021214740FFFFFF5F59EE5EC0000000608E244740FFFFFFDF9BF85EC0FFFFFF5FCC1E4740010000007EFF5EC0000000E0BC27474000000060FB045FC0000000603B224740000000601C045FC000000020E9514740000000A064015FC0FFFFFF5FB74A474001000040C0005FC00000004022314740FFFFFF9FC5F55EC0FFFFFF3FC7334740000000801FFC5EC0000000009B3D4740FFFFFF5F1BF95EC0000000E0704147400100004036FD5EC0FFFFFF1F074F47400000006036FB5EC0000000C02C564740FFFFFF7FBDF55EC0000000C0F65B4740000000A03BF95EC0000000A0615F474001000020AE025FC000000020A75B4740FFFFFFBFBE055FC0000000A0565D4740FFFFFF3F73065FC0000000601365474000000080CD085FC0000000E038734740FFFFFFBFAF065FC00000006044744740FFFFFFFF9E065FC000000020EA6F474001000000C3015FC00000004077694740000000E0EB025FC0000000A097714740000000A0EDF35EC000000040697B474000000040A7FF5EC000000060007D4740000000801E025FC000000020FF834740010000E01B075FC0000000A07C854740000000C0490A5FC0000000C003774740000000A0400C5FC00000000066954740FFFFFF7FBA0E5FC0000000E03BA34740000000405C145FC000000040BAAC47400000000042165FC0FFFFFF3F78C34740000000C0D3175FC000000080C9D14740FFFFFFDFE41E5FC0000000807BE7474000000040BE265FC0FFFFFF3FDDEF4740FFFFFF9FCF2E5FC0000000E039134840FFFFFFFF0B2D5FC000000020B91D4840FFFFFFFFCF2D5FC0000000E05A304840FFFFFF1FFB235FC000000020C22D4840000000805AFF5EC0000000C065144840FFFFFFDF50D95EC0000000603C0E4840FFFFFFBFCDC75EC0000000C00F13484000000060E6BA5EC000000020140C4840FFFFFF9F1BBB5EC0000000E092084840000000C0BFB55EC0000000A0101148400000006020B15EC00000006074124840000000404EB35EC0000000E0F10A48400000004042AA5EC0000000806BF54740FFFFFF9FBFA95EC0000000C0ABEE474000000080A7AF5EC0FFFFFFFF92E747400000004077B25EC000000000C0E6474000000040BAB35EC0000000C0BAED4740FFFFFFBFE1B65EC000000000ECE94740FFFFFF5F7AB95EC0FFFFFF1F1BD64740FFFFFF7FD0BE5EC00000004086CD4740000000A035C75EC0000000406DBA474000000060C7C95EC0FFFFFF3FA3AC4740FFFFFFDF96C05EC00000000036AD4740FFFFFF5F3FB55EC0000000A025B84740000000803CC25EC00000002099AD47400000008021C75EC0000000A095AF4740FFFFFFBF9AC15EC00000004010C24740000000E09ABA5EC000000080B1CE4740000000801BB05EC00000008097D44740000000E031AE5EC000000020E8E04740FFFFFF9F08A75EC000000020D3EC4740000000402AA75EC000000020DBF74740FFFFFFDFF5A15EC0000000406FF44740000000E03A9E5EC000000020A9E0474000000040B2A75EC0000000403CD947400000002074A55EC0000000C022C94740FFFFFFFF74A35EC0FFFFFF3FB6CA474000000040A7A25EC0000000E0EEC24740000000C034A05EC000000060F2C04740FFFFFF3FA9A35EC0FFFFFF7F03B34740FFFFFF9FBEA25EC0000000C0E2AF4740000000A091A55EC000000020C4AA4740010000A052A35EC0000000204AA44740FFFFFF1F13A55EC00000006033A04740FFFFFF3F0FA75EC00000000094A54740FFFFFF5FC3A65EC000000000A8A2474000000040B4AC5EC00000000069A547400000002029A85EC0000000C009B34740FFFFFF5FB7A85EC0000000A00AB347400000002061AF5EC0FFFFFF9FBAAB4740000000802EB15EC00000006017A24740FFFFFFDFFCAD5EC000000080959C4740000000C0A3B05EC0000000A0D2944740FFFFFF1FBAB45EC0000000C0149E4740FFFFFFDF69B15EC0FFFFFF9F34AB47400100008042B35EC0000000C032AE47400000008043B85EC0FFFFFF3F53A64740000000A04EC75EC000000020A59A4740000000C01DC55EC0000000008D8B474001000000EDC15EC000000020EC8C4740FFFFFF5F00BB5EC0000000A029864740FFFFFFBF7BB25EC000000020229047400000004086AE5EC0FFFFFF5F938A4740000000C0B9AC5EC0000000E09B8C4740FFFFFFDFCBA55EC00000002012974740FFFFFFBFE3A15EC0FFFFFF3FD1A4474000000000E7A25EC0000000A081A8474000000020109B5EC0000000403CA147400000002010995EC00000004092A34740FFFFFFFF2E9C5EC00000002091A64740000000C0DA9A5EC0000000C0D5A84740000000C0BE945EC0000000A018AC47400100006062945EC0FFFFFF1FF5B14740000000A00C995EC0FFFFFF7F55C14740FFFFFFFF61985EC0000000403CCC474000000000789A5EC0000000C009D54740000000002B995EC0000000201EE34740000000A04E935EC0000000A0A6F94740FFFFFFDFA58E5EC0000000E011FC4740000000C0CE8D5EC0000000C0F9004840000000207E975EC0FFFFFFDF6C104840FFFFFFDFA5995EC0FFFFFFBFD91C48400103000000010000000C00000000000040DEBD5EC000000020D4384840FFFFFF1F03C65EC0000000A0633D4840000000A023CA5EC0000000A0D1424840FFFFFF5FCACA5EC00000000008484840FFFFFFBFF1C85EC0FFFFFF7FD94F4840FFFFFF1F8EC65EC000000020E54D484000000000B1C05EC000000040614748400000006079C05EC0000000C05644484000000040DEBD5EC00000006078434840FFFFFFBF57C15EC0000000A0BB4148400000004020C15EC0FFFFFF3FB13E484000000040DEBD5EC000000020D43848400103000000010000001400000000000080D7AE5EC0000000006F234840FFFFFF5F84AA5EC000000060CF3248400000004099A65EC0FFFFFFDFD53348400000000091A15EC0000000C01D294840FFFFFF7FC0A15EC0000000E04F244840000000A0D2A75EC0000000A0F4254840000000A0C4AE5EC000000040E01C48400100008004A75EC0000000A06E1A4840FFFFFF3FE0A25EC000000080DC09484000000040AD9F5EC000000080100C4840FFFFFF3F3D985EC00000006023044840FFFFFF5FAA965EC0000000A066FB4740FFFFFF7FAF985EC000000040CEF3474001000020429C5EC0000000E088F54740000000601A9E5EC0000000C06CFE474000000000CCA25EC000000020DEFB474000000000DFA65EC0000000000C044840000000006FAC5EC00000000037174840000000A01EB15EC000000020081C484000000080D7AE5EC0000000006F234840 Washington 53 0.64322916666666666667 0.63574879227053140097 0.60612939841089670829 0.59467455621301775148 0.60063897763578274760 0.65147058823529411765 0.67867036011080332410 0.65552995391705069124 0.63119072708113804004 0.73207547169811320755 0.68298109010011123471 0.64070107108081791626 0.74226804123711340206 0.76765083440308087291 0.92215944758317639674 0.95497185741088180113 0.86313868613138686131 0.79738190096755833808 0.84971751412429378531 0.89427312775330396476 0.88365650969529085873 0.82939759036144578313 0.84758028041610131162 0.82311222361284939508 0.84429914180629342051 0.86004140786749482402 0.83735654926790660863 0.79010549290651145871 0.80498220640569395018 0.83522519223727572318 0.84039548022598870056 0.83253588516746411483 0.83333333333333333333 0.84170854271356783920 0.84153846153846153846 0.84034107615407233167 0.85905665643315657270 0.87377387712958182757 0.85005959475566150179 0.86653162277740265586 0.84526511244068495977 0.82612966601178781925 0.82660377358490566038 0.83043707214323328067 0.85114564973561929178 0.86878027300748568912 0.90247271722613620666 0.91071655041217501585 0.89898989898989898990 0.90869120654396728016 0.90830370978032996081 0.87732132808103545301 0.85849260728452939055 0.83412735690800509964 0.83068548642319134122 0.79755403868031854380 0.78548206576773437084 0.77462820640782513225 0.74446220598310116465 0.71937351605781647020 0.72372756522076210412 0.74902752842609216038 0.77805904031567583665 0.76539200279378383098 0.75717856901560705358 0.75720555482772173457 0.74742542335743575297 0.75546725621414913958 0.75337116529947185077 0.76449855815443768023 0.77333333333333333333 0.77578740157480314961 0.75813051396674471960 0.76642631065419671117 0.77678192367564507760 0.77726676886844853252 0.76002722130066777253 0.75248860813955762423 0.75620248470239198962 0.76269072769953051643 0.77023285800970873786 +15 0106000020E6100000020000000103000000010000005C010000000000E0AEA055C000000020D85342400000008050B155C0000000A076534240000000205DC455C0000000604C5342400000008036C755C0000000A05C534240FFFFFFBF2ED655C0000000001A5342400000004000E955C0000000E0945242400000008062EC55C0FFFFFFFF7C52424000000000A0F655C0000000C01B524240FFFFFF5FB9F755C000000000AE55424000000060900456C0FFFFFF1FFE564240FFFFFFBFA00256C0000000E0944A4240000000403E0256C0FFFFFFFFE144424000000020BC0256C0000000E08D3F424000000020BE1F56C000000060C33F424000000000CF2056C000000040EF3F4240000000E0E13356C0FFFFFFBFDE3F424000000020E23456C000000000FB3F4240000000E0233556C000000040F93F4240000000A02E5656C00000008053404240000000A08A5A56C0000000C055404240000000C0C25A56C0000000205A414240000000A0ED5756C0FFFFFF3FDF4E424000000060445756C00000000017504240000000A0E85556C0000000207E50424000000020A05456C0000000409E4F4240FFFFFF9F235256C0000000C0A1494240000000A0764F56C0000000C0DD484240000000A0714D56C0000000807B4A4240000000C0CE4C56C000000060CE50424000000080554B56C0FFFFFF9F95534240000000C0BD4A56C000000000F6554240000000C0A34C56C0000000A04F5B424000000020904C56C0000000201C5D424000000040574B56C0000000006661424000000000B04956C00000004028614240000000C0074856C0000000E04E624240000000C00D4856C0FFFFFFBF6D65424000000020854A56C000000040F7664240000000001A4B56C000000040296A4240000000E0A74A56C000000040F56B4240000000204B4856C000000080E76E4240FFFFFFBFB74656C000000060187A424000000080DB4656C0000000201D7D4240FFFFFF9F4F4856C0000000807A7E4240000000A09D4956C0000000E0CC7F424000000040284B56C0000000804A834240000000E0D94A56C0000000E038884240000000C05D4956C0FFFFFF7FED8B4240FFFFFFFF794756C0000000805A8E424000000080294456C000000040CA97424000000020903F56C0000000202A9C424000000020AE3B56C0000000C0F49B424000000020403756C000000080E1994240000000C0C62F56C00000004078934240FFFFFF9F4D2F56C00000004012924240000000400E2C56C00000002055914240000000604F2756C000000040F58D4240FFFFFF1FCB2356C00000000052894240FFFFFFFF1A2156C0000000604A884240FFFFFF9F671F56C000000020BA884240FFFFFFDF831E56C0000000003C89424000000080D41C56C000000040A18C4240000000800A1B56C0000000A015944240FFFFFF3FD81C56C000000060539A424000000060172056C000000000FFA0424000000080B92056C000000040FFA54240000000E0ED1D56C0000000004CB34240000000E0DE1A56C000000020CCB54240000000C0FC1656C00000004064B4424000000000F31356C000000060AFB84240FFFFFFDF9F0556C0FFFFFF7FF6BC424000000080940456C0FFFFFF1F68C1424000000040960856C000000080B2CA4240000000A0160A56C00000000072D0424000000020330A56C00000006091D44240000000808D0856C000000000B2D9424000000060A30456C0FFFFFF9F21DE4240FFFFFFDF460256C0000000A020E7424000000060B70056C00000004091E64240000000805BFD55C0000000E059E342400000002022FC55C00000004056E64240000000A0E3FA55C0000000C0A3E74240000000C040FA55C00000004056EB4240000000E0F4FB55C0000000E005F0424000000020CEFB55C000000080BBF34240000000E0FFFA55C0FFFFFF1FBEF542400000006089F955C0000000C057F64240000000C0DBF655C0FFFFFF1F09F2424000000040B6F455C00000004069F04240000000A03DF055C000000080F4F24240000000809AEE55C00000006080F24240000000A066ED55C0000000C029F342400000004080EB55C000000020D1F24240FFFFFF1FD2EB55C0000000E00CEB424000000020B5E955C00000004000EA424000000080E2E655C0000000E000EC4240000000E0FDE555C0FFFFFFFFB3EE4240000000A00FE655C00000004003F24240000000C022E855C00000006032F6424000000000ADE655C0000000804DFC4240000000804EE055C0000000E031F5424000000020F2DC55C000000080DEF7424000000080CDD855C0000000A0ABF7424000000020E0D355C0000000E063F24240000000A074D155C00000008075EF42400000004083CE55C000000060AEEC4240FFFFFFFF3FCB55C00000000057EB4240000000001ECA55C000000080D8E94240000000C070C855C0FFFFFF7F14E54240FFFFFFBFCFC655C0000000C060E442400000006090C455C0000000A04EE74240000000C055C255C0FFFFFFDF37F44240000000A0D7C055C0000000005DF64240FFFFFF5F4CBF55C0000000C01CF74240000000009FBB55C00000000010F84240FFFFFFFF9AB955C00000000011FA42400000000040B755C0000000A051FE424000000060E2B455C0000000A0E9FE42400000008061B355C00000008047FD4240000000E03EB055C0000000A0FBF2424000000020A6AE55C00000002081F24240000000E01AAC55C0FFFFFFBFB5F44240FFFFFF7FCBAA55C0000000C0E1F44240000000C042AA55C0000000A085F3424000000080ECAA55C0000000A027EE4240FFFFFFBF9EAA55C00000002075EC4240FFFFFF3F51A955C0FFFFFFDF47EC4240000000E058A755C0FFFFFF3FD0ED4240000000004BA655C0000000C0E3F54240000000403CA555C0000000407AF6424000000080A1A255C000000040F2F54240000000E074A155C000000080C2F642400000004015A155C00000006099F84240000000C0F9A155C0000000A063FE424000000060C8A155C000000080620243400000002039A155C0000000A0030643400000006033A055C0000000609A06434000000040569D55C00000008090074340000000C0519C55C000000020B8094340000000A0529C55C000000060580B4340000000E05B9E55C0FFFFFF5F4A0E434000000060C09D55C0000000208610434000000080F69C55C0000000C099104340000000A00F9A55C000000000D80D434000000060329955C000000000C60F434000000020059655C0000000A02D114340000000E0779555C0000000606D12434000000020F69555C000000040E7134340FFFFFF9FC69855C0000000A07F15434000000060DA9855C0FFFFFF5FED184340000000E0519755C0FFFFFF9FBB18434000000040DD9555C000000040AF164340FFFFFF7F0D9355C0000000003B13434000000060A79255C0FFFFFFBF090A434000000040C69155C00000000070074340000000C0239055C00000002034054340FFFFFF9F338C55C0000000A043024340000000A0B88655C00000002071014340FFFFFF3F608355C000000040BDFB4240000000A0068255C0000000A016FF4240000000C06D8055C0FFFFFF7F37004340000000005A7D55C0000000A08101434000000000947B55C00000006059044340000000E08B7A55C0000000804B08434000000000607A55C00000000008174340000000408D7655C0000000E0861E4340000000A0C17555C0000000405B234340000000209F7355C0000000409F244340000000E0517255C0000000202324434000000040CE6F55C00000006097224340000000809C6B55C0000000608326434000000080DF6955C000000040392B434000000040316955C0000000801A31434000000020366755C0000000602A394340000000A0766055C000000060553C4340000000E0D95D55C00000008051424340000000A0AC5B55C0000000E0BA44434000000060B85A55C0FFFFFF5FDC474340000000202A5B55C000000000DA4A4340FFFFFFBF095D55C000000020E958434000000040975C55C0000000A0C55C434000000040C45A55C0FFFFFF7F825E4340FFFFFF7F715555C000000040545E4340000000405F5155C0000000C0455F434000000020224D55C0000000800E594340FFFFFF7F4D4A55C000000080F958434000000040A94755C0000000E0665B434000000060624455C0000000E00B604340FFFFFF9F9B4155C000000040D261434000000040713E55C000000000EA634340000000C0673455C0000000608C654340FFFFFF3FC43455C0000000A0CD6A434000000060663255C000000020EC6E4340000000607A3255C00000008031714340000000E0683355C000000020D5724340000000E0063755C0000000207473434000000000053855C00000004066744340000000400F3855C0000000C0B9764340000000E02A3655C0000000602F7A434000000060683555C0000000A0C97D4340000000A0083655C000000020BD80434000000000163855C0FFFFFFFF3384434000000080F63855C0000000C079864340000000A0C03855C0000000805188434000000080FC3455C0000000C0438D434000000020F03355C0000000A01F8D4340000000008F3255C000000060B18D4340FFFFFF1F8C2F55C0000000402D92434000000000B92A55C0000000E0768B434000000060DA2755C00000008095894340000000C0F52555C000000080FC884340000000A0FB2055C0000000A00C8C4340000000C07E1F55C000000020BC8D4340000000807A1C55C0000000604E8E4340000000603F1B55C000000040D68A434000000000DE1A55C0FFFFFF1F0D864340000000400C1955C0000000409184434000000040221655C000000000D5844340000000600E1455C000000020CB814340000000A0921255C000000080E4784340000000E0BD1055C0000000C06D75434000000020100F55C000000040EF6F434000000020A40E55C00000000004684340FFFFFFFF500B55C000000040EB64434000000020B10555C0000000C0F9614340FFFFFF9F720355C0000000C0BF6143400000004095FD54C0000000A087634340FFFFFF3F68FA54C0000000800261434000000060E3F654C000000020575F4340000000609BF554C0000000801C5B43400000004098F254C000000080CD584340FFFFFF9F4CF154C0000000A04B53434000000040A0ED54C00000006057514340000000606EEB54C0FFFFFF1F784F434000000040F9E954C0000000E0D84F4340000000602BE954C00000008061514340FFFFFF5F88E854C0FFFFFF5F1B554340000000E094E754C000000060C5564340FFFFFF7FB4E154C0000000C017594340000000A002E054C000000000545843400000008009DD54C000000020F4544340000000E0C6D754C000000080D4534340000000A020D554C000000080E2504340000000C081D454C000000060A14D4340000000C09FD354C0000000A0514C4340000000A091D254C0FFFFFF1F5C4C4340FFFFFF5F76D154C0FFFFFF9FF94D4340000000E0AFCF54C000000060E24F434000000080A6CB54C0000000C00C4E4340000000002BC954C000000000444F43400000004020C754C0000000C01655434000000020E7C354C000000060C357434000000020BBC154C0000000A0725B4340000000E03EBE54C0000000C01A5C434000000060F8BA54C000000000885F4340000000A0FCB854C0000000C0105F434000000020E4B754C0000000E0055C4340000000E053B854C0000000E073574340000000800CB754C0000000207F53434000000060A7B654C0FFFFFF3FD94C434000000040EFB454C0000000A029494340000000C05BB354C0000000A052474340000000E07DAF54C000000040C84643400000004086AC54C0000000000045434000000040DFAA54C000000080434043400000008049A754C0000000C07D3C4340000000E08CA554C0000000C0CA344340000000804BA654C000000020272F434000000080A2A454C0000000A0632743400000008021A554C0000000406E254340000000E0C7A454C000000000C120434000000000B6A554C000000020661F4340FFFFFF3F72A754C0000000A08E1E434000000040D5A654C000000080CC184340000000E049A754C000000000C9164340000000006DA954C000000040AD154340000000205CA954C0FFFFFF1FB812434000000020F9A554C000000060100E43400000006096A154C0FFFFFF3FFE01434000000040759E54C000000080E7FC424000000040BD9E54C0000000A026FA4240000000E0959F54C000000040A1F842400000008005A054C0000000A009F64240FFFFFFDF039C54C00000008087F2424000000020FC9A54C000000040A6EF4240000000E0FA9954C000000040E3E7424000000040C29554C00000000064E44240000000E0749454C0FFFFFFDF10E1424000000020179554C0FFFFFFBF3DDF4240000000C0ED9254C0FFFFFF5FA0D54240000000C0438F54C0000000E00DD4424000000040288D54C000000060DCCF424000000060E28B54C000000000FED1424000000080348A54C0FFFFFF9FF6CB424000000080718854C00000004093CB4240FFFFFFBFCE8854C0000000E0EEC8424000000000638954C0FFFFFF9F6CC84240FFFFFF3F218954C00000002057C7424000000020698554C0000000802BC6424000000080928354C0FFFFFFDF3AC3424000000080278354C00000008092C6424000000040B28154C000000080E4C34240000000C0827E54C0000000C085C54240000000406C7D54C0000000E0F9C3424000000060809254C00000002002A74240FFFFFF7FA79654C00000002055A1424000000020FC9954C00000008013A04240FFFFFFDF35A354C0000000C081994240000000805CA454C0000000E0CE984240000000A007AE54C0000000C0118E4240000000002DAE54C000000000E88B42400000000065AD54C000000020A68942400000006017AE54C0000000206D8842400000006051AE54C0000000A05684424000000060FDB354C0FFFFFF5FB4804240FFFFFF9F77B754C000000020BC7C42400000008016B754C0000000004E774240000000C033B854C00000008061724240000000E0DBBC54C0FFFFFF1F976E4240000000E0FDC254C000000000EA6D4240000000205BC454C000000080EA6C42400000008036C854C000000060B8634240000000C0F7C754C0000000602360424000000020DFC854C000000060B75E4240000000600ACD54C000000060F95D4240FFFFFF1F93D454C000000020CF5A424000000060B3D854C0000000C01458424000000020DFD954C0000000000C564240000000C075DD54C000000020B4544240000000A0FBE154C0000000A0A8544240000000A066E954C000000060F64E42400000008037EB54C0000000A09F4C42400000004086EC54C000000000C64A424000000020E2FB54C0000000C0AC4B4240000000C06F0055C0000000E0C64B4240000000A04A1055C0FFFFFF1F354C424000000020701055C0000000A0364C4240000000200B3255C0000000C0704D4240000000A0A13255C0FFFFFF9F7C4D4240000000A0E73F55C0000000207A4F424000000040715155C0000000201250424000000060355355C00000000022504240FFFFFF7FFE5B55C0000000201F4F4240000000A0457255C00000006035504240000000A0C27E55C0000000200851424000000080BC8C55C0FFFFFFDF55524240000000C0969A55C00000006050534240000000E0AEA055C000000020D85342400103000000010000000B000000FFFFFFDF1F6256C000000000C23F424000000080496456C0FFFFFFFF65424240000000A05C6456C0000000E04C454240000000C0976356C0000000206447424000000080F16156C00000006043484240000000608F5F56C00000002091474240000000C0D35E56C0FFFFFF7F1D464240FFFFFFFF2A5E56C00000006045434240FFFFFFDFD35E56C0000000E099404240000000E0735E56C000000060D03F4240FFFFFFDF1F6256C000000000C23F4240 Kentucky 21 0.34114583333333333333 0.31400966183574879227 0.33030646992054483541 0.31213017751479289941 0.32747603833865814696 0.34264705882352941176 0.36703601108033240997 0.33870967741935483871 0.35932560590094836670 0.37358490566037735849 0.33926585094549499444 0.31158714703018500487 0.33934707903780068729 0.34531450577663671374 0.44005021971123666039 0.47967479674796747967 0.48844282238442822384 0.47011952191235059761 0.48870056497175141243 0.54790748898678414097 0.51966759002770083102 0.47710843373493975904 0.52148349163274536409 0.51606174384647476012 0.53330608908868001635 0.53374741200828157350 0.53264740799366838148 0.52273554019643506730 0.53060498220640569395 0.56499450750640790919 0.56638418079096045198 0.55809979494190020506 0.56903353057199211045 0.57223618090452261307 0.58646153846153846154 0.57982946192296383417 0.59559028746860173039 0.60118740320082601962 0.59618593563766388558 0.61174881836596893991 0.61295646791829997937 0.62554027504911591356 0.63981132075471698113 0.65209759522555731087 0.66415638519468033969 0.67620725084397475415 0.68144771377262052770 0.69270767279644895371 0.69960973370064279155 0.69366053169734151329 0.69638136906389572509 0.66170914060615805129 0.65705012621709340065 0.64342749781923102731 0.62402683714159123995 0.62923777019340159272 0.61599957362895059425 0.59861263599161592973 0.58378625256907969856 0.56525180155787895197 0.56603481024925378920 0.57914422501496110114 0.60458623385325540707 0.60485419940632093592 0.60181744476724545639 0.60638019127472815407 0.60146492628415813691 0.60214507648183556405 0.59599393190246095067 0.59684396026914450497 0.59127226463104325700 0.59778543307086614173 0.58695806428723479742 0.60200851954974893506 0.60797679135808292844 0.59745828612055366704 0.59293522181106716005 0.57481469892241800794 0.56449100686074541072 0.57405369718309859155 0.59257433252427184466 +47 0106000020E610000002000000010300000001000000F5000000000000A0E6EF55C0000000A0167B4640000000C0C0F555C0000000E0B2764640000000C02DF555C000000020CA6F46400000008015FF55C0000000C0385C4640000000C0EAFE55C0000000E0B056464000000000D80056C0000000E0CE5146400000002071FE55C0000000E0434C4640FFFFFF9FD30056C0000000A0A54F464000000080940256C0000000802549464000000000D5FD55C0000000608B444640000000A048FB55C0000000C0024546400000008098F755C000000000F74E464000000080E7F055C000000060705246400000008051EE55C0000000C03A584640000000C051E755C000000080A16A4640000000405FE355C0FFFFFFBFF86C4640FFFFFFFF4CE355C00000000059694640000000E0C0DB55C0000000A00F724640000000E082D755C000000080E1674640000000A01ED455C0FFFFFF5FB9654640000000E0DED755C000000040A5564640000000E04CDE55C00000006058444640000000E064E255C000000020F7294640000000801AE155C0000000407F164640000000003CE955C0000000E0850C46400000006077EE55C0000000A06BF24540FFFFFF1FF8EC55C0000000C02AD64540FFFFFF1F4DF255C0FFFFFF1FEDC545400000004062F355C0FFFFFF1FB7BA45400000000004F855C000000040E6AD4540000000A0F1F855C0FFFFFF1F3E994540000000000AF755C0FFFFFFDFB5894540000000E014F955C0FFFFFF1F4C834540000000E086F555C000000040777B45400000002077F455C0FFFFFFBFB76B4540000000406EF055C00000000086634540000000E0A6F255C0FFFFFFDF535545400000002007F355C0000000E09B3E454000000000760C56C0000000A0AB3E4540000000C0101356C0FFFFFFDFF83E4540000000A0372D56C000000000AD3E4540FFFFFFFFF43056C000000000D63E4540000000E0193C56C0000000A0D43E4540FFFFFF1F015756C000000060BB3F4540000000C0A15956C0000000C0AD3F4540000000606A7556C0FFFFFF5F71404540000000C01B7B56C0000000A086404540000000E0E09A56C0000000001241454000000060DAA856C0000000C0324145400000008009A856C0000000E0A7434540000000E0E6A856C0FFFFFF9F214745400000002083AA56C000000060234945400000006075AC56C0000000C0A751454000000000B6AF56C0FFFFFFBF18544540FFFFFF5F1DB956C0000000C0D056454000000080D5BA56C0FFFFFF9F20574540FFFFFF7FF0BF56C000000020815A4540000000003AC456C0FFFFFF9F595F4540000000E03DC556C0000000A045644540FFFFFF9FF8C556C0FFFFFFBF8B6F454000000040E5C856C00000002084764540FFFFFFBFBBC956C0000000A02B8045400000004037CA56C0FFFFFFBF648A4540000000C0C7CA56C0000000A09C8A45400000008051CA56C000000060E49245400000004069C456C00000008003A14540000000403EC456C0FFFFFF1FEEA345400000000004C556C0000000E01AA84540000000A052CB56C0FFFFFFDF4EAD4540000000E0ADCC56C0FFFFFFBF6DAF4540000000807DCD56C00000002048B64540000000E016CF56C0000000A07BBB4540000000C04CCE56C0000000601BC045400000002063CF56C0FFFFFF1F3DC64540FFFFFF1FE7CE56C000000060A9CC45400000004087D056C000000080B3D64540000000E08FD056C00000008078DC4540000000E00FD056C0000000C0E0E4454000000000ADD256C0000000E071EC4540000000C0E2D756C0000000C03EF94540000000A03FDB56C0000000002AFE454000000040CFE156C00000006062044640FFFFFFBF6AE456C0000000A07A0446400000004081E656C0000000E03A054640000000C0BBE956C0FFFFFF3F910846400000004032F056C0000000E091114640000000404FF656C0000000007A184640000000C0DDF856C0000000C0F62046400000002005FB56C000000080E9244640000000C00BFB56C0000000A0A5284640000000C013FC56C000000020692B4640FFFFFFDF38FE56C0FFFFFF9FA82E4640000000A0D50557C00000002033354640000000802E0D57C0000000601E38464000000060EE0F57C000000060663A464000000000FA1257C0FFFFFFDF003F4640000000C07F1457C0000000E02F454640000000E0CD1557C000000060C4464640FFFFFFDF932057C0000000E09F49464000000040F62657C0000000201F4E4640000000C0542857C00000008043524640000000202A2F57C000000020585B4640000000608B3357C000000040835F464000000060B13057C000000080EE6A464000000060E63057C0000000C05E6E464000000000633157C0FFFFFFBF23734640000000003D3057C0000000E01F754640000000E0F82F57C0FFFFFF9FC477464000000040153157C0000000002280464000000080D13057C0000000E0D5824640FFFFFFBFFA3257C00000000067884640FFFFFF9FB12F57C000000000788E4640000000A0A92F57C0000000C006944640000000C0CA3057C000000000E497464000000060553057C000000040309B4640000000C0C42F57C00000000019A6464000000060422D57C0FFFFFFFFBBA8464000000080D12B57C0000000807AAE4640000000C0812957C0FFFFFFDF9FB24640000000E0432957C00000004041B8464000000020E52957C00000000046BA464000000080DA2B57C0FFFFFFFF2BBC4640000000A0962E57C0FFFFFF5F0DC6464000000000C43057C0FFFFFF1F3BC84640000000C06D3557C0000000E01EC84640FFFFFF7F1A3857C0000000A018CA4640000000E0A63857C0000000408FD2464000000020073757C000000020F5DA4640000000E0563557C0000000408FDD464000000080D93157C0000000A0B6E1464000000060E82F57C0000000402EEB464000000020F82E57C0000000E029EC4640000000C02F2D57C0FFFFFF7F0CF24640000000E09F2A57C00000006037F54640000000C05B2357C0000000E0C4F94640000000A0852157C0FFFFFFFFC6FD4640000000E0931D57C0000000A099FD464000000000301B57C0000000804503474000000060581757C0000000401602474000000060251657C000000040E602474000000080F01457C0000000A049074740000000E0811257C00000008061094740000000E07A1257C0000000600D144740000000A0761257C0000000E040354740FFFFFF7F5F1257C00000000055544740000000A05F0D57C0000000A0CE52474000000040210657C0FFFFFF5F105F474000000000410057C0FFFFFFBF8857474000000020F6FA56C0000000A010574740FFFFFFBF8EE356C0000000E0E26047400000008023B756C000000020ED794740000000208EB156C000000080CC754740FFFFFF9FBEB156C0000000600C7147400000008044BB56C0FFFFFF9FF34A474000000000C1AE56C000000020A8524740000000C09AA256C0000000E0354B4740000000001D9A56C000000000CA48474000000080A99856C00000004015454740000000E0109456C0000000609B46474000000080579356C000000040AD45474000000040339356C0000000A03643474000000040419156C000000060E2424740000000C0869056C0FFFFFFBF21414740000000C0868D56C000000000D040474000000060518A56C0000000A0A138474000000060108956C0000000006D324740000000405C8756C000000020BF2E4740000000A0228756C0000000E0942B4740FFFFFFBF327B56C0FFFFFFFFEB264740FFFFFFFF604656C0FFFFFFFFA5124740000000000D3F56C000000020DB0C474000000040343B56C0000000406D09474000000020793356C0000000C06F034740000000C0CB3256C0000000E0A804474000000020C03156C0000000202E04474000000000773156C000000060B6024740000000607B2E56C0000000C0CA034740000000C0052D56C0FFFFFF7F6D02474000000020582B56C00000008095024740000000C02F2956C0000000C028FF464000000040622756C0000000C040FF4640000000E03B2656C0000000E0FD01474000000080D02456C0000000002701474000000020162356C0000000E079024740000000A0FD2056C0FFFFFF3F63024740FFFFFFDF9C1F56C000000020AA014740000000A0F41E56C0000000A0E5FF4640FFFFFF5F111D56C0000000601A00474000000020D11956C000000040E2FD4640000000E0AA1756C00000002050FF4640FFFFFF5F8E1456C0000000C0BEFB464000000020231356C0FFFFFF7F22FB464000000040731056C000000000CAFB464000000040C00D56C00000006056F94640FFFFFF1F860B56C0000000600EFA4640000000A09E0956C000000000DAF74640000000E01E0756C00000002092F6464000000080FF0556C000000040D8F54640000000E01E0656C00000002028F24640000000C02D0456C000000000D5EF464000000040C90756C0FFFFFFBFDEEA4640000000004F0856C0000000A0E3E84640000000C0AB0556C0FFFFFF5F52E54640000000004C0356C0000000C0A0E44640000000405BFF55C000000080C5E546400000000005FE55C0000000601CE2464000000080E7F755C00000004018E0464000000040E7F555C00000008079DC4640000000A04AF355C00000002010DB46400000002044F355C000000040C6D94640FFFFFF1FC0F155C0000000E091D7464000000000F9F155C0000000A085D64640000000A048F455C0000000C02CD54640000000E077F455C000000040C6D34640000000C0A8F155C0000000807ECE4640000000E098F155C0000000A0DACC4640FFFFFFFF50F255C0FFFFFF5FC6C84640000000E005F555C0000000C0C8C846400000008085F355C000000020B4C546400000006083F255C000000080E2BF4640FFFFFF5F10F455C000000000B6BB4640000000C00CF755C000000000FAB84640000000E05CF655C0000000A0FCB34640000000408BF855C000000040D5AE464000000060EDF755C0000000C059AE46400000004094F755C000000000A1AF4640000000C02AF755C00000008062AF4640000000C0D5F555C0FFFFFF7F4FAC464000000040FCF455C000000060DEAD4640000000A0A2F055C0000000A02CAD46400000008020EC55C0FFFFFFFF15B246400000004030E955C00000002052AE4640000000C04BE955C0000000A091AC46400000004014ED55C000000060D8A24640FFFFFF3F1FED55C000000040A19F4640000000400DEE55C0000000404F9E4640000000602DEE55C0000000C0189B4640000000201CEF55C0000000E07B994640FFFFFF1FB1EE55C0000000A09B964640000000A00DEB55C0FFFFFF1F02924640000000C08BEA55C000000000F68D4640000000E031E555C0FFFFFF9F1D8C46400000002094E755C0000000E045874640000000E0B1E755C0FFFFFF1FFA7E4640000000A0E6EF55C0000000A0167B464001030000000100000017000000000000C033C255C0000000802CA54640000000E01CBF55C0000000E03AA6464000000020EDBD55C0000000C0C19E4640FFFFFF3FB8BF55C000000040F59B464000000040E1C255C000000040E09F4640000000209FC155C0000000A032934640000000E01AC555C000000040DB924640000000E0DDC255C0000000003A8C4640000000C09AC555C0000000C0CC8B4640000000E05CC555C000000040D28646400000006032C755C0000000604A884640000000006ECB55C0000000C0CC7D4640000000C0CBCA55C00000004077774640000000C027CD55C000000060C66F4640000000E0E7D355C0FFFFFF3F3E664640000000A03BD855C0000000403B6B4640000000C0F0D955C000000060A274464000000040E4D555C0000000C0F2814640000000E022D255C000000080BC86464000000080C5CE55C0000000E068964640FFFFFF1F61CB55C000000080D69346400000008038C455C000000000F3A54640000000C033C255C0000000802CA54640 Wisconsin 55 0.58420138888888888889 0.56811594202898550725 0.53234960272417707151 0.53550295857988165680 0.53194888178913738019 0.55882352941176470588 0.63850415512465373961 0.59677419354838709677 0.58061116965226554268 0.63773584905660377358 0.57063403781979977753 0.53261927945472249270 0.57731958762886597938 0.55584082156611039795 0.66101694915254237288 0.69355847404627892433 0.71897810218978102190 0.68924302788844621514 0.73220338983050847458 0.78964757709251101322 0.76842105263157894737 0.72578313253012048193 0.78516508367254635911 0.75052148518982060909 0.75153248876174908051 0.73457556935817805383 0.74198654531064503364 0.72389959985449254274 0.73309608540925266904 0.75979494690589527646 0.78566384180790960452 0.77170198222829801777 0.75739644970414201183 0.75753768844221105528 0.75630769230769230769 0.76918553366656865628 0.77839799051074518560 0.78084667010841507486 0.75804529201430274136 0.77447670492910195814 0.77388075097998762121 0.78251473477406679764 0.79962264150943396226 0.80656485869756011936 0.82198365646531004647 0.82518714222809335095 0.83727034120734908136 0.84590995561192136969 0.85135445362718089991 0.84785276073619631902 0.84595752438246285662 0.81686630758099525685 0.79379733141002524342 0.77782996712071395021 0.76245331983036901070 0.74982935153583617747 0.73788839737781804615 0.72512226769138636590 0.70143868463119433661 0.67484483692256425209 0.67058185060278326937 0.67923399162178336326 0.69653426646316494807 0.69397590361445783133 0.69721640429700695899 0.71069697366697235687 0.70657651735687231978 0.70369263862332695985 0.69642656478255983818 0.70076364413115454448 0.69694656488549618321 0.69468503937007874016 0.68972776082688805317 0.71007353466124080817 0.72307656021510448606 0.70437023706369837144 0.69382416741099910680 0.67539553712866446326 0.66454663452623771556 0.67110475352112676056 0.67650182038834951456 +22 0106000020E610000001000000010300000001000000A6010000000000A0D91C56C0000000A0796F3F40000000A0CF1B56C000000020E61E3F40FFFFFFBF771B56C00000008021003F40000000E0B31A56C0FFFFFFFF7BBC3E40FFFFFF9FB01956C0FFFFFFBFB2643E40000000C08C1956C0000000A0495A3E40000000E0B51D56C0000000A06C533E40000000C0F92456C0FFFFFFBF6B613E4000000040BA2B56C0FFFFFF5F95573E40000000A0EB3756C0000000A0196E3E4000000040883B56C000000080A66A3E4000000040AF3856C000000040E9653E4000000040AB5156C0FFFFFF7F8C503E40FFFFFFBF805156C0000000E0475F3E40000000A0705556C0000000A056613E4000000000E15656C0FFFFFF7F765D3E40FFFFFF3F625456C0FFFFFF7F93513E40000000A0985A56C0000000209C413E4000000060095C56C0FFFFFF9F65333E40000000A0B96456C000000060DA313E4000000060096756C0000000E0C03D3E40000000E0CC6656C0FFFFFFDF643F3E4000000060C86756C0FFFFFF9FBB413E40000000800E6856C000000000484A3E4000000040E76856C0000000C0AE4B3E4000000000C56856C001000040C84F3E40FFFFFFFF956756C000000040D5523E4000000060B06756C000000040DE573E4000000080896856C0FFFFFFBFE85A3E40FFFFFF5F626956C0010000A0E75A3E4000000020E56956C0FFFFFFBFFD603E4000000040386B56C0000000A05E663E40FFFFFFBF396B56C000000000F6713E40000000A0BC6B56C00000002068763E4000000040736C56C0FFFFFF7FCE773E4000000000856C56C0FFFFFFBF617A3E40FFFFFFBFA36D56C001000080317B3E4000000000046E56C0FFFFFFBFEF7E3E4000000040DD6E56C0000000C0657F3E40000000C0537056C000000040F3833E4000000060747156C000000000128D3E40000000209C7256C0000000C0C28D3E40000000C07E7456C0000000C0C39F3E40000000C08C7356C0FFFFFF5F36A63E40FFFFFF1F6F7456C0000000A0ABA63E40000000C0157556C000000000B2AB3E40000000A01A7656C0FFFFFF1F83AA3E40000000E0757556C001000040FAAC3E40000000A0DE7556C0000000A0E8AD3E4000000060D77556C0FFFFFFDF4AB33E4000000060297556C0000000A01EB43E40000000C0037656C0FFFFFF1F55B63E4000000080797556C000000020ABBA3E40FFFFFF5FB17456C0000000209DBB3E40000000E0D47456C0000000E011BE3E40FFFFFFBF0C7456C0000000A07BBF3E4000000080C67456C0FFFFFF1F1FCA3E4000000020E57356C0FFFFFF3FBFCF3E40000000C0F97256C0010000C0FDCF3E4000000000F27256C0FFFFFFFF33D43E4000000020957156C00000008091D43E40000000A03C7256C00000004084DC3E4000000080027156C0FFFFFF5FA5DC3E40000000E0D87056C0FFFFFF1F79E63E40FFFFFF5FCA6F56C0FFFFFF5F11E73E4000000040B96F56C0FFFFFF9FF0E83E40FFFFFFBF827056C000000060BDEB3E40FFFFFFBF626F56C00000006009ED3E40000000E0E66F56C0010000201DF33E4000000020536F56C00000006018F73E4000000060106E56C0000000E0ADF93E4000000060826E56C0FFFFFFBF03FC3E40000000A04E6E56C0000000E057003F40000000A0E36E56C0000000E0DB013F40000000C0577556C0000000E0CA013F4000000020989056C000000040D7013F4000000000FD9556C00000008055013F4000000040B9A256C0FFFFFFFF8C003F40000000A0D9A356C00000002064003F400000006018B556C0000000A059003F4000000060A4C356C0000000C00F003F400000006076CB56C0FFFFFF9F12003F40000000A075E856C0000000C04D003F40000000802BE856C00000002004033F40000000A0A6E456C00000000050083F400000006053E356C000000000D80E3F40000000A026E456C000000040FB143F4000000000AEE756C0FFFFFFFFA4203F4000000000D9E556C001000080A52D3F400000004074E656C0FFFFFF9FB8363F40FFFFFF9FC2E856C000000080AD3D3F400000002030E956C0000000805C453F400000000095E856C0000000800B473F40000000A0A2E356C00000002032453F40000000E015E156C0000000006C483F40FFFFFF9F26E056C0FFFFFFBF764C3F40000000605CE056C0FFFFFFDFC2523F4000000020BBE256C0FFFFFFDFC8583F40000000A0D5E256C000000060635E3F400000008034E256C0FFFFFFBFF0613F40000000E0CAE356C0FFFFFFDF57633F40000000C053E456C000000020D66B3F40000000C059E356C0FFFFFF7FD46E3F400000004038E256C0FFFFFF7F256F3F40FFFFFFDF2DE256C0FFFFFFBFB8683F400000002077E156C00000000055643F4000000060A7DF56C0FFFFFF3F05603F4000000020C4DE56C000000020AB603F40000000C0CFDD56C00000000066673F400000008048DE56C0FFFFFF9F926B3F40000000807FE056C000000040F2733F400000004068E056C00000008088863F40000000802EE056C0FFFFFFFFE2883F4000000000B8DD56C000000060EE8A3F40000000001ADB56C0FFFFFF3F05903F4000000060DDD956C0FFFFFF5F10963F40000000C093DA56C0000000A0389A3F4000000060CFDF56C000000060AC9A3F40000000808FE056C0000000002D9F3F40000000007EE056C0FFFFFFDFC9A43F4000000080D9DF56C00000004096A63F40000000C043DD56C00000000076A03F400000006021DA56C0000000E017A03F40FFFFFF7FE6D856C0FFFFFF5F89A73F4000000040D2D856C0000000C06DB73F40000000805BD756C0FFFFFF5F67C03F4000000060B4D756C000000000D9C03F4000000040BFD556C0FFFFFF1F25C23F40000000A014D256C0000000E0EDBF3F4000000060E3D056C00000002075C23F40FFFFFFFFCDD056C0FFFFFF5F15C63F400000006083D556C00000000072C33F40000000407DD756C0000000E04CC53F400000004037D656C001000000B1CB3F40000000406CD556C0FFFFFF9FE3D73F40000000406AD356C000000080ECDC3F40000000E0B8D256C0FFFFFF3F80DC3F40FFFFFF3FB5D156C0FFFFFF9FCED33F40000000200AD056C0FFFFFF9F93D13F400000000071CF56C0FFFFFF5FD1D53F40000000E0EED056C0000000808CDD3F40FFFFFFBFA3CF56C000000020E6E03F4000000040E4CC56C0FFFFFFDF0EEA3F40000000206FCA56C0000000A000FD3F4000000080E9C656C0FFFFFF5FE8FD3F40000000E0A1C456C000000000B2024040FFFFFF3FAEC456C00000006018044040000000808FC556C000000080960540400000002027C956C00000002076084040FFFFFF3F51C956C000000060F70A4040FFFFFF1F0AC856C0000000A04B0B404000000020B7C556C0FFFFFF9F460740400000002029C456C0000000A07E07404000000060B9C456C000000000080C404000000020A1C256C000000020C80D4040000000E0B8C056C0000000E028114040FFFFFFBF3EC056C000000060AA1540400000000090C156C000000040BF154040000000807CC356C0000000C09E17404000000060E6C256C0000000604F1440400000008075C356C0000000C0331140400000000050C556C0000000A00413404000000020F5C656C00000006047114040000000C004CA56C00000004088124040000000A0ADCA56C0000000C003164040000000A010CA56C0000000C0591A4040000000C06BC756C0000000A0E01C4040FFFFFFBF6FC656C000000060811B404000000060A0C556C0000000E0BE1D404000000060FFC356C000000080CA1D404000000060AAC256C0000000A0ED1F4040000000C08ABE56C000000080981C40400000002025BE56C0000000C070224040000000E0F6BE56C0000000C0C0244040FFFFFFDFC4BE56C00000008019264040000000A02FBE56C0000000A0DF264040000000C078BB56C00000006011264040000000E0A5BA56C0000000A014274040000000C005B856C0FFFFFF5F9830404000000060A6B856C000000040D0304040FFFFFF1F8EBA56C000000060462C40400000002003BF56C0000000609E2D404000000060EDBF56C0FFFFFFBFE02E4040FFFFFFDF96C056C0000000A0D3324040FFFFFF3F14BE56C000000060A235404000000080CABD56C0FFFFFF1F533840400000006032BF56C0000000C0FA394040000000E0B8C156C000000020A1384040000000A0BEC356C000000040373940400000002088C756C000000020D13F40400000008099C756C0000000A014434040000000A0AEC556C0000000C0394640400000008061C256C0000000409C3F40400000008042C156C000000000D33E4040000000C07DBF56C000000060653F404000000040AFBF56C0000000807D414040FFFFFF9F09C456C00000008065454040FFFFFF5FB0C456C0000000E0E247404000000080C4C356C000000020C7494040000000A011C256C000000000954A404000000000EABF56C0000000A0C74E4040000000A0C0BF56C000000000205040400000002081C056C0000000204352404000000000C2C156C0000000C01C524040000000600CC356C000000020A34E40400000008015C456C0000000209E4D4040000000801AC756C0000000C07B4C4040000000605CC956C0000000808A52404000000080EEC856C0000000E0E254404000000020C7C356C0000000409C5C4040FFFFFFDF40C656C000000040EC5F4040000000E0BBC856C0000000E024604040FFFFFF7F01CA56C0000000809F6140400000000019C956C0000000E0AC6B4040FFFFFFFF16C556C0000000E04670404000000080D6C456C000000040137A4040FFFFFF5FEEC556C0FFFFFF5F6A7E40400000000028C756C000000080617E40400000008078C856C000000020D47C4040000000E050C856C0000000E0FE77404000000080C7C856C0000000007A75404000000060DCCA56C0000000C0D773404000000020B1CC56C00000004003754040000000E062CD56C000000000CA774040000000005CCA56C0000000A009804040FFFFFF5F60CA56C000000040AB8140400000008048CA56C000000060C78240400000002007CA56C0000000E02C85404000000060EFC756C0000000000D864040000000E089C756C0000000A064884040FFFFFFFF64C956C0000000809A8B4040000000C031CC56C0000000207E8E4040FFFFFF1F83CC56C0000000C0FA914040000000405ECB56C0000000A03C934040000000E0C3C756C0000000E0C69040400000008022C656C000000020929240400000004086C556C000000060AF94404000000060E3C556C0000000A0E39C4040000000A07DC356C0FFFFFFBF759F40400000006096C256C00000000016A440400000000072C356C0000000E093A5404000000080E2C456C0000000006CA540400000000096C656C000000020E69F404000000020D7C756C0000000605EA240400000006012C956C0000000E043A94040000000E058C856C0000000E000AE4040FFFFFF1FD8C656C0000000205BB24040FFFFFF3F09C556C0000000C080B44040000000E0EFC356C00000002045B74040FFFFFF1FE7C356C0000000A0E1BA4040FFFFFF7FB8C456C0000000A0ACBB4040000000C07DC556C0000000E03EBB4040FFFFFF1F58C656C0000000601BB5404000000000CFC856C0000000C0C6B14040FFFFFFBFD6CB56C00000008027B240400000000011CD56C0000000C007B54040000000A0B9CC56C0FFFFFFBF07B64040000000605AC856C0FFFFFF3FB8B8404000000060A9C756C000000060F6B94040000000009DC756C0000000401EBC4040000000403EC856C0000000E020BF4040000000008FCA56C00000000082C14040FFFFFFBF25CB56C0000000208FC04040000000A0FDCA56C000000000BBBB404000000020A2CB56C00000000044B9404000000000E7CE56C0000000C0C2B84040000000A089CE56C000000060CFBA4040000000004ACD56C00000004098BC4040000000C08BCB56C0000000A089C14040000000C0B3CB56C000000020FDC240400000000010CD56C0000000C0EDC44040000000C0A9CD56C00000000007C540400000008091CE56C00000008036C74040000000C082CE56C0000000E094CB40400000000005CC56C0000000208FC94040000000E0C0CA56C000000020E3C9404000000040A4C956C0FFFFFF5FDECE4040000000C0E1C956C0000000408AD14040000000C023CD56C0000000C0C0D54040000000E0C4CD56C0000000A069D84040000000208CCD56C000000060BFDA40400000008073CA56C0000000C0F1DB404000000040BFC756C0FFFFFF3FB9D64040000000805CC556C0000000A0CFD44040000000E06BC256C0000000A073D74040000000207CC256C0000000804DDA40400000000099C356C0FFFFFFDF13DC404000000040B1C656C0FFFFFF5FA7DA4040000000A040C856C00000004031DB404000000000D5C856C0000000A092DC40400000002025C956C000000000CBE2404000000060C4C856C0000000E0D9E3404000000020BFC656C00000006062E34040000000C03FC456C0000000E0ACE4404000000020C7C256C0000000C080E24040000000002EC156C0000000E0C9E14040FFFFFF3FB4BF56C000000020C2E2404000000080FABE56C00000000086E440400000004062BF56C00000004042E64040FFFFFFBFD8C156C0000000E085E840400000002082C356C0000000A0F8EB404000000080EDC356C0FFFFFF7FFCEE4040FFFFFF7F2EC156C0000000E0D8F7404000000060D8C456C0FFFFFF1FC1FC4040000000C0AFC556C0000000C04AFF4040FFFFFFDF75C456C0000000C0C7004140000000E0FCC156C0000000602BFE4040000000609CC056C0000000A0C9FE4040FFFFFF3F04C056C0FFFFFF9FF3FB40400000000028BF56C000000000FCFA404000000020BFBD56C0000000C0D5FB40400000002083BD56C0000000C04BFD4040000000E06ABE56C0000000404FFF4040000000A04BBE56C00000006068014140FFFFFF7FD4BC56C00000004001044140000000E0B9B856C000000000360541400000000072B756C000000000EC0C41400000000001BA56C0000000E0260D4140000000C04DBC56C0000000E01E104140000000A003BD56C000000060F11341400000006073BB56C0FFFFFF5FC3174140000000E02EB656C0000000C0E2124140000000200CB556C00000004007134140FFFFFF3FAEB356C00000002045154140000000C0AAB456C0000000C063184140FFFFFF9FF9BA56C0000000E0371A4140FFFFFFDFC1BB56C0FFFFFFDF091E4140FFFFFF7F6BBB56C00000006006204140FFFFFF1F42B756C0FFFFFF7F0D1C41400000000035B556C000000060611D4140000000C0B9B456C0000000E07F234140000000609CB356C00000000051264140000000C0B8B256C000000000652641400000004087B056C000000080B5234140FFFFFF3FDBAF56C000000000AB28414000000040C2B056C0000000C0942E4140FFFFFFFF56B056C000000020A32F4140000000C0FFAB56C0000000205E304140000000C097AB56C0000000E0832E4140000000C01EAC56C000000080FA284140000000407AAB56C000000060B3284140000000A019AA56C0000000A03D2A41400000004010AA56C000000020D62E4140FFFFFF7FA4A656C0000000E0C93341400000006010A556C0000000806C37414000000000C3A456C0000000E01A3A414000000060C3A556C0000000808D3F41400000006024A556C0000000A096424140FFFFFF1F34A456C00000004029444140000000A060A256C0000000C08B454140000000A0F5A156C0000000801F474140000000A0F7A456C000000040684D4140000000A0A1A556C0000000405C5041400000004013A556C000000060A3524140FFFFFF5FE8A356C000000000A35941400000000080A256C000000020CD574140000000000BA356C0000000606E534140000000607EA256C0000000C085514140FFFFFF5F90A056C000000020AC514140000000A0D69D56C0FFFFFF5F0556414000000020149E56C000000000255A414000000040DEA056C000000060DD594140FFFFFF3F21A256C0000000E04B5B4140000000400EA356C0000000C029654140000000A0BEA156C00000006056674140FFFFFFDF06A156C0000000001E674140FFFFFFFF14A056C0FFFFFF3F1965414000000040EB9F56C00000004005624140FFFFFFFF15A156C0FFFFFF9FCA5F41400000006048A056C0000000E06B5D414000000060199F56C0000000E0085D414000000040E49C56C0000000A0DF5E4140FFFFFF3FBA9C56C00000004060614140FFFFFF7FDE9D56C0000000605B66414000000000EC9C56C0FFFFFFBFA0694140000000C0619E56C000000000CA6D4140000000201D9E56C000000020C2704140FFFFFF9F099C56C0000000606E714140000000C0619B56C0000000C0B26F4140FFFFFF3FBF9B56C0000000E0EC6A414000000020079B56C0000000C0876A414000000000DA9956C000000020A76B4140000000E0D99556C000000060276E4140FFFFFF1FA99456C000000080D56C4140000000A04C9356C000000000076D4140000000202A9356C0FFFFFFFFB66E414000000020F69256C000000000FC704140000000C0119156C0000000E0C0724140FFFFFFBF8A8F56C000000080DA754140FFFFFF3F7B8F56C0000000002E78414000000000E28F56C0000000C091794140000000202B9356C0000000E03E7D4140000000E08A9356C0000000C016804140FFFFFF5FE66D56C0000000C0E47F4140000000E05F6956C0000000001580414000000040E85556C0000000A0F67F4140FFFFFF1FAD4C56C0000000001A80414000000080644056C0000000C004804140000000E0FF3356C0000000004D804140FFFFFF3F3D3256C00000006065804140000000C0841856C000000060A280414000000000821656C0000000C07A804140000000C0690C56C0FFFFFF1F8F80414000000000280956C0000000A01177414000000060F70656C0FFFFFF3F2E734140000000A0C90556C000000020A1724140FFFFFF1FBA0856C0000000A04A4A414000000080AD0956C0FFFFFFDF8A3B414000000080B90A56C0000000607A29414000000080C10C56C000000060900B414000000080F70C56C0000000E08D074140FFFFFFDFE10F56C0000000000EDF404000000040921156C0000000A0F3C44040FFFFFFBF811356C000000080F6A4404000000040B91556C000000020627E4040000000C0451656C0000000405A76404000000000341956C000000060484A4040000000A03F1B56C0000000009027404000000020031C56C0FFFFFF5F221D404000000060441E56C00000006083E33F40FFFFFFBFC31D56C0000000E0BBB33F40000000A0D91C56C0000000A0796F3F40 Mississippi 28 0.24826388888888888889 0.19516908212560386473 0.19863791146424517594 0.18786982248520710059 0.20926517571884984026 0.25588235294117647059 0.24515235457063711911 0.26382488479262672811 0.23603793466807165437 0.25283018867924528302 0.22803114571746384872 0.20934761441090555015 0.26374570446735395189 0.28177150192554557125 0.33458882611424984306 0.39337085678549093183 0.38260340632603406326 0.34775184974388161639 0.37853107344632768362 0.44162995594713656388 0.39058171745152354571 0.37108433734939759036 0.38489371325192220715 0.37797246558197747184 0.38414384961176951369 0.38426501035196687371 0.41353383458646616541 0.38232084394325209167 0.37829181494661921708 0.42328817283046503112 0.43961864406779661017 0.42276144907723855092 0.43458251150558842867 0.42776381909547738693 0.46123076923076923077 0.45780652749191414290 0.47111359196204298074 0.47470314919979349510 0.47699642431466030989 0.49448570785505289219 0.49680214565710748917 0.51886051080550098232 0.54094339622641509434 0.56310338774793751097 0.57891363563531485339 0.57771906649053280493 0.58088133720127089377 0.60329740012682308180 0.60365013774104683196 0.59366053169734151329 0.59693738036642056330 0.56885601736473993086 0.56985214569058781103 0.55700194591692947729 0.54528767643521741882 0.53828213879408418658 0.52880669402547567020 0.51367401936320990119 0.49842429778488239324 0.48714958137209980422 0.48610303523665542505 0.49236983842010771993 0.51394110858802069761 0.51374192421861358477 0.52253226133369366935 0.54202148565439538844 0.53792218361661501862 0.53907743785850860421 0.53053713900438251489 0.53436398590195450176 0.52641221374045801527 0.51656003937007874016 0.52560372761891246245 0.53639846743295019157 0.55108731543940751922 0.54435779510764513764 0.55548466675173323125 0.53343242133260321123 0.53350639718153161506 0.54267532276995305164 0.55593901699029126214 +46 0106000020E6100000010000000103000000010000008D01000000000080D7CE53C0000000E07C3D43400000004072D153C0000000A0F5374340000000A04DD453C000000020CD3443400000002024DF53C0000000A0233B4340FFFFFFFF59E253C0FFFFFF3FDF464340000000E020E953C000000080CE4B4340000000A0DBEA53C00000006068464340000000209DEA53C0FFFFFFFFA44243400000002058EC53C0000000C007404340000000E0CBEB53C0000000200E374340FFFFFFBF18EE53C0000000208132434000000060EAEE53C000000020052D434000000020E9F053C0000000A04B2D4340000000403CF353C0FFFFFFDF372843400000006064F353C0000000603D264340000000E059F253C0FFFFFFBF7A24434000000060CEF253C0000000A05F2243400000004035F553C0000000200520434000000000A6FA53C000000020EE164340000000A046FA53C000000040CC14434000000000E0FB53C000000000830F4340000000C06CFB53C000000040350D4340000000804BFD53C0000000609B084340000000A0DEFD53C0FFFFFF7FED044340000000C00B0054C000000000B0FE424000000080850354C0000000A04EFA4240FFFFFF3FD40654C0000000800FF5424000000020990754C00000006011F24240FFFFFFFF400A54C00000000045F0424000000020090B54C00000008016EE4240000000E0FE0A54C0FFFFFF5FE2EB424000000020550E54C0FFFFFFDFAFE64240000000E0200E54C0FFFFFF9FADE3424000000040501054C0FFFFFFFFE8E04240FFFFFFFF031054C000000040EBDC4240000000A0691354C0000000C05DD7424000000040F01254C0000000C0EFD5424000000060861354C0000000C078D3424000000000461354C000000060F9D14240000000004C1054C000000080FED1424000000080060E54C0000000E0E3CF424000000080C80F54C00000000063CC4240000000C0481454C00000002086C8424000000040DF1454C00000006042C44240000000E0BE1354C0000000809DC3424000000020FB1154C0FFFFFFFF9FC44240000000E0701254C00000004069C1424000000040411654C0000000C0DABE4240000000608D1654C0000000C0ECBC4240000000A0DD1854C0000000C098BB4240000000003D1B54C0000000E0A6B7424000000020661E54C0FFFFFFDF1AB64240FFFFFF3F2B1F54C0000000A084B74240000000003D1F54C0000000C0F0BA424000000020932054C000000040CABC424000000000C02254C0FFFFFFFF0ABC4240000000C0402654C00000004014B9424000000080252D54C000000060B2B14240FFFFFF5FB72E54C0000000A040B2424000000020C72F54C0000000609DB1424000000080DD2F54C0000000C081B0424000000040D83054C0FFFFFF7F86AF4240000000604B3154C0FFFFFFDF6AB14240000000202A3354C00000000021B2424000000020303354C000000040DBB4424000000040723654C00000000030B64240000000E0293854C0000000E0BCB14240000000C04F3654C0000000C0E7AC4240000000A0C23654C0000000E06DAB4240000000E0CC3B54C0000000608FA6424000000020F53D54C00000008055A5424000000080A33E54C000000000EFA54240000000C01C3F54C0000000002FA74240000000A09B4154C0000000C099A44240000000E0044954C0000000E02CA3424000000080474E54C000000080BB9E4240000000A0F85354C00000006094A5424000000060F95654C0000000005FAB424000000020085954C000000020D0A7424000000040D35954C00000004029A44240000000206F5E54C0000000208DA04240FFFFFF9FB95F54C0000000A059A04240000000805D6054C000000020FC9D424000000000A36354C000000000669A4240FFFFFF9FA06A54C0000000C0369A4240000000E0EB6C54C0000000001F9E4240000000A0456F54C0FFFFFF5F0CA0424000000020217054C0000000A0D5A24240000000A0BD7254C0000000C0BDA44240000000E0317454C000000040C4A34240FFFFFFFFB27554C0000000C087A4424000000040F77654C00000002049A74240000000604B7754C0000000E0A4A94240000000A06D7954C0000000C094AB4240000000A0537B54C00000002091AF4240000000E0EF7A54C0000000002CB54240FFFFFF3F417F54C000000080B5BB424000000040807E54C000000040CCBD424000000080AE7C54C0FFFFFFDF17BF424000000000E17B54C000000020D6C04240000000406C7D54C0000000E0F9C34240000000C0827E54C0000000C085C5424000000040B28154C000000080E4C3424000000080278354C00000008092C6424000000080928354C0FFFFFFDF3AC3424000000020698554C0000000802BC64240FFFFFF3F218954C00000002057C7424000000000638954C0FFFFFF9F6CC84240FFFFFFBFCE8854C0000000E0EEC8424000000080718854C00000004093CB424000000080348A54C0FFFFFF9FF6CB424000000060E28B54C000000000FED1424000000040288D54C000000060DCCF4240000000C0438F54C0000000E00DD44240000000C0ED9254C0FFFFFF5FA0D5424000000020179554C0FFFFFFBF3DDF4240000000E0749454C0FFFFFFDF10E1424000000040C29554C00000000064E44240000000E0FA9954C000000040E3E7424000000020FC9A54C000000040A6EF4240FFFFFFDF039C54C00000008087F242400000008005A054C0000000A009F64240000000E0959F54C000000040A1F8424000000040BD9E54C0000000A026FA424000000040759E54C000000080E7FC42400000006096A154C0FFFFFF3FFE01434000000020F9A554C000000060100E4340000000205CA954C0FFFFFF1FB8124340000000006DA954C000000040AD154340000000E049A754C000000000C916434000000040D5A654C000000080CC184340FFFFFF3F72A754C0000000A08E1E434000000000B6A554C000000020661F4340000000E0C7A454C000000000C12043400000008021A554C0000000406E25434000000080A2A454C0000000A063274340000000804BA654C000000020272F4340000000E08CA554C0000000C0CA344340000000A0D5A454C000000060B0334340FFFFFFFF0CA354C00000004041334340000000E0AF9F54C0000000A0EF334340FFFFFF9F8F9A54C00000006014374340000000E0459954C000000060D536434000000060139554C0000000208F384340000000A01E9454C0000000E0893B434000000000919254C0000000603D4A434000000080589154C0FFFFFF9F224C4340FFFFFFBFAE8D54C000000020D94A4340000000E0CC8B54C000000040274C4340FFFFFF5F1F8B54C0FFFFFFDFE8504340FFFFFF5F1A8C54C000000080C256434000000060C88B54C000000080E85A434000000060E18D54C0000000A0B1634340000000A0A98C54C000000020FB664340000000E05B8954C0000000C05A6B4340FFFFFFBFEC8854C0000000E01C734340FFFFFF5F7C8654C0000000A0DB79434000000020738554C0FFFFFF3F127D4340000000C0C08354C000000020977E4340000000E0C08254C0000000C0CC81434000000000FD7F54C0FFFFFF7FF1814340000000C06B7E54C000000040187F434000000020067C54C000000040DC7E4340000000E0637B54C000000000FA7D434000000020857954C00000008050774340000000C0A57B54C00000004084724340000000C0957A54C0000000E03271434000000040247954C0000000A0CA6F4340000000007C7754C0000000405C714340000000E0D37554C0000000200A78434000000020BB7454C0000000C064794340000000C0227254C0000000A034764340000000E0CB7054C0000000800D774340000000400A7254C0FFFFFF1FF67B434000000020A77154C0000000E02482434000000020127454C0000000C0A285434000000040C37454C0000000C07D88434000000020767454C000000020D9894340FFFFFF1F567254C000000000E1894340000000C03C7054C0000000401D8C4340FFFFFF9FAB6F54C0000000201A90434000000060947054C0000000807C96434000000040496E54C0FFFFFFDF499B4340FFFFFFDFAC6C54C000000020279C4340000000A0236C54C0000000A04CA1434000000020BB6A54C0000000209DA2434000000060A96454C00000002007A24340000000C0AE6354C00000000092AA4340000000809C6254C00000002023AD434000000040C55D54C00000008011B44340000000E0AD5C54C0FFFFFF1F9AB44340000000E0C85B54C000000020F6B34340000000A0115854C0000000203DAC4340FFFFFF1FB25554C0FFFFFFBF41AD434000000000305254C00000002089B14340000000E0374F54C000000000B7B1434000000040684E54C0000000A042B4434000000080D44C54C0000000A039B5434000000020914B54C00000006007B8434000000020814754C0000000E0DDBB4340000000604C4654C0FFFFFF3F89BF434000000040674254C0FFFFFFDF2BC4434000000060184254C000000000A4C54340FFFFFFDFF63E54C00000002076CA4340000000C0B23B54C0FFFFFFBFADCD4340000000C06A3A54C000000040BBCD434000000000673854C0FFFFFF5FDFCF434000000000DE3754C000000060C7D4434000000020453754C00000004013D74340000000604F3554C00000008006DA434000000060473554C00000004000DC434000000020D33654C0000000C03DDE434000000000BD3754C00000000045E14340000000406F3454C0000000E08AE74340000000E0DE3454C0FFFFFFBF77EB4340000000201E3354C0000000A0A6ED434000000060A03254C0000000A0A6EF434000000020FD3354C000000060D1F34340000000C0B63354C0000000E039F5434000000020F53254C0000000E0BAF54340000000202C3154C000000000E5F44340000000C0943054C0000000A0E9F5434000000020D93054C00000006035F94340000000204D2F54C000000020E0FD434000000080422F54C0000000408E044440000000E0F12C54C0FFFFFFDFB6134440FFFFFF9FDE2C54C0000000A084154440000000C0702B54C0000000A0D7184440000000C09E2954C000000020701F4440FFFFFF5F5A2754C00000002062234440000000C0B32654C0FFFFFFBF3027444000000040012754C000000040C52F4440000000E0482854C0FFFFFF7FBD31444000000000322854C0000000E0F6324440000000C0872654C0FFFFFF1F803D444000000080072854C00000002090404440000000A08D2854C0FFFFFF7F0245444000000000CE2A54C000000040BB484440FFFFFF5FBF2A54C000000060814A444000000080CD2854C0000000E0944E444000000000272754C0FFFFFF1F5C4F4440000000A0C62454C000000020D64E4440000000E06B2154C0FFFFFFBF8D51444000000080922154C0000000A0463D444000000080852154C0000000609433444000000020AE2154C000000020CB144440000000609C2154C000000080E902444000000040972154C0FFFFFFFFAAFA4340000000E0902154C0000000E04DDC434000000060791B54C00000002021DC434000000080C8FA53C0000000005DDC4340000000A0FBF053C0000000A061DC4340FFFFFF1FCCDE53C0000000602FDC4340000000A05DDF53C0000000C040994340000000E087DD53C0000000C0489B4340000000C0C4DC53C0FFFFFF5F229B434000000020A5D853C00000000075A24340000000E02BD653C0000000E05FA54340000000A0E9D253C0000000C074A6434000000080ECD153C0000000409EA9434000000080AAD053C0000000009DAC4340000000E072CA53C0000000E05AB24340000000C022CA53C000000080F9B44340FFFFFFDF6CC853C0000000205EB5434000000040ADC653C0000000403EB94340000000A034C653C0000000E075BB4340000000C0B5C653C00000008042BC43400000004089C453C0000000C041BC43400000006023C453C0000000802CBE43400000008024C353C0000000A0EABD4340000000C01FBE53C0FFFFFF7F1EB843400000004029BD53C000000040EDBA4340000000A0BFB753C0000000204AC34340000000E0A7B553C0FFFFFFDF17C84340000000C0A2B353C0000000C08AC84340000000A0A6B453C000000040F5CA4340000000601EB353C000000020C3CE43400000002019B353C000000040BCD043400000004078B153C0FFFFFF9F73D24340000000E024B153C00000000032D04340000000A0E3AE53C0FFFFFF7F3DD04340000000C0C4AE53C0000000E08BCF43400000000023AF53C000000000EACD43400000004089B153C0000000E0FECC4340000000E0BFB053C00000004075CA4340FFFFFF1FE9AE53C000000080CCC9434000000060DCAD53C0000000209DC7434000000000ABAA53C0FFFFFF1FB7C44340FFFFFFDF8FA953C000000020DAC4434000000040CAA853C000000060D2C34340FFFFFF3FB2A653C0FFFFFF3F8EC443400000000020A453C0FFFFFF7FAFC243400000004094A053C00000006035C34340000000A0D19E53C0000000408AC24340FFFFFF5F309D53C0000000604EC44340FFFFFF7F8D9C53C0000000402CC6434000000020F39A53C00000000050C64340FFFFFFBF929D53C0000000C055CA434000000080DB9C53C0000000C0DACB4340FFFFFF7FDF9953C0000000E033CB434000000080A89B53C00000006078CF434000000040A29853C0000000C0A4CE434000000020309853C000000000CCD04340000000E0D99653C000000080EED04340000000C0489653C0000000E0FBD14340000000807D9153C00000000025CF434000000000839053C0000000C00ED2434000000020B08E53C0FFFFFFDF48D4434000000060958E53C00000004042D64340000000C0178D53C00000002082D64340FFFFFF3FBA8B53C000000000E7D84340000000200E8653C00000000077D6434000000060B58153C000000000B7CF434000000000B67F53C0000000E0A7CC4340000000A0BA7D53C0000000C03CCE4340000000207F7C53C000000040FFCA434000000020E37B53C000000080C1CB434000000020A97C53C0000000A0B5CE4340000000C0167C53C0000000A01ECF434000000080D37953C0000000A04ACC4340FFFFFF7F067953C0000000A0E0CC4340000000C0E07853C0000000A0E8CE434000000080C57653C0000000A010CD4340000000A0EE7553C0000000C079CD434000000000C57553C0FFFFFF5F4CC94340000000809B7653C0000000A05DC8434000000040AB7853C0000000C03CC8434000000000F97853C0000000606CC7434000000080AB7753C000000040DDC54340000000C05A7753C0000000E0DCC1434000000040067653C00000002013C44340FFFFFFDF7C7553C00000000044C34340000000C0117553C000000060BBC34340FFFFFFBFD57453C00000004087C14340FFFFFF3F487653C0000000803EC0434000000080D47453C00000002035BF4340000000C0657153C000000000BFBF4340000000A02F7353C00000008088BD4340000000E0437253C0000000A0C0BA4340000000807C7353C0000000E044BB434000000000F37253C000000060B4B94340FFFFFFBF847353C0000000404FB84340000000205D7353C0000000E052B7434000000020777053C00000008068B64340000000806E6F53C0000000A0A0B3434000000080336F53C000000040B3B24340000000A06A7053C0000000A06EB0434000000020BA6F53C0FFFFFF5F1DAE4340000000204B7053C0000000C053AB434000000020067053C0000000E0D1A94340FFFFFF7F936E53C000000040AAA84340000000A09F7053C0FFFFFFDF6BA44340000000602F7153C0000000A08B9F434000000020917353C0000000002799434000000040807453C0000000A020924340000000802E7553C0000000E0E7904340000000A0268253C000000020FDA14340000000C0B48E53C0000000C00CB24340000000E0BC9153C0000000E030B64340000000A0429653C0000000C07BBA4340000000A06E9653C0000000C0BBB0434000000060689753C00000008048AE434000000060079653C0FFFFFFDFE8AC4340000000E0D49553C0FFFFFF9FB1AB4340FFFFFFFF7B9A53C0000000C0F3A04340000000C08F9953C000000040579F434000000000189B53C000000020249B4340FFFFFF5F289B53C00000008048994340000000C0C49953C0000000A0D2954340FFFFFFDF929B53C0000000C00293434000000020B09C53C000000020398F4340000000C0129F53C0000000C0508E4340000000A01EA053C000000060FA8B4340000000E05CA253C0000000A04C874340000000E01FA453C0000000207C844340000000802AA353C0FFFFFF1FFE824340000000206CA353C000000020C58143400000006055A653C000000020CD7B43400000002064A853C0000000A0637D4340000000406CA953C000000020A8794340000000208DAB53C0FFFFFF1FF67543400000002008AE53C000000020D3734340000000A05CAE53C00000004011774340FFFFFF3F3BAF53C000000020EF76434000000000F8AF53C000000020A8744340FFFFFF9FC5B253C000000060A7704340000000403BB453C0FFFFFF7FB46A4340FFFFFFDF79B753C000000080B36143400000008036BF53C0000000005F6C4340000000E02CC253C00000006061664340000000E085C353C0000000C02F654340000000A0A2C353C0FFFFFF3F876143400000004099C553C000000000845A4340000000C0AEC553C0000000005D54434000000080C3C753C0FFFFFF7FF2544340000000C02BC853C0000000803D54434000000080D7CE53C0000000E07C3D4340 West Virginia 54 0.39930555555555555556 0.39420289855072463768 0.40408626560726447219 0.38017751479289940828 0.41373801916932907348 0.46029411764705882353 0.46675900277008310249 0.44930875576036866359 0.44046364594309799789 0.46540880503144654088 0.43159065628476084538 0.39629990262901655307 0.42783505154639175258 0.39345314505776636714 0.46390458254865034526 0.51282051282051282051 0.54014598540145985401 0.52646556630620375640 0.58305084745762711864 0.61123348017621145374 0.56675900277008310249 0.50891566265060240964 0.53459972862957937585 0.52023362536503963287 0.52145484266448712709 0.50724637681159420290 0.52156707558369608231 0.53728628592215351037 0.56725978647686832740 0.56792383742218967411 0.56391242937853107345 0.55536568694463431306 0.54930966469428007890 0.55339195979899497487 0.57015384615384615385 0.58041752425757130256 0.59335752162991906224 0.58621579762519359835 0.57616209773539928486 0.57911321179383299572 0.57726428718795131009 0.61237721021611001965 0.63735849056603773585 0.64630507284535720555 0.64508892805640121775 0.65419051812710993689 0.68711147948611686697 0.69486366518706404566 0.69478879706152433425 0.68537832310838445808 0.67742229514173730745 0.65696599405096872739 0.63945185719437432384 0.63336241025296920083 0.60927906829546173808 0.59254835039817974972 0.58284922453765389330 0.57211298532787703364 0.54578670929435944280 0.52934560753113675178 0.52444082645268829709 0.54529473369239976062 0.56654133938875032573 0.56287759734590536057 0.56496182690358759543 0.57031966461417529150 0.56070992581463048174 0.55467256214149139579 0.54466794021800202270 0.54058528249492683969 0.53348600508905852417 0.53924704724409448819 0.55188154875943139620 0.57359415530330073059 0.58082928440020755696 0.56619787153680375036 0.56088639360299434307 0.55500361801576281462 0.55199332468014092342 0.57337514671361502347 0.60381902305825242718 +16 0106000020E6100000040000000103000000010000000202000000000060456D57C000000080473D3E40FFFFFFDFBF6C57C0000000A0214C3E40000000A0B46E57C0FFFFFFFF0E4E3E40FFFFFF5F967057C00000006043573E4000000000997057C001000040A95A3E4000000080DC6F57C0000000A00E5E3E4000000000517057C0000000E0BC613E4000000020866F57C0000000A0A8683E40000000A02D6E57C0000000C0D76E3E40000000A0946C57C00000006050713E40000000E0046D57C00000004067763E40000000A0AB6C57C0FFFFFFFF54783E40FFFFFF1FC06D57C0FFFFFFBF167D3E4000000000446D57C0FFFFFF9F097F3E40FFFFFF9FBC6D57C0FFFFFF1F4F813E4000000040266D57C000000000DA853E40000000400F6F57C0FFFFFFFFA68B3E40000000C0F16D57C0FFFFFF7F72913E40000000A0F06D57C0FFFFFF7F5E963E4000000000616C57C0010000404D993E4000000040FB6A57C0000000800B993E4000000020556C57C0000000C0A49D3E4000000040D06B57C0000000C0989F3E4000000020586C57C0FFFFFFBFD9A33E40000000E0636B57C0FFFFFFDFC2A33E40000000403D6A57C00000008040AC3E40000000E0866757C000000040D2AF3E40FFFFFFBF316757C000000020D8B53E40FFFFFFDF896757C00000004088BB3E40000000C0E36657C00000000065BB3E40000000C0946757C0FFFFFFFFEBBE3E4000000080736557C0FFFFFFDFADC53E40000000603D6557C00000008052CD3E40000000603E6357C0000000600ED43E40000000A08F6357C0FFFFFFBFA3D73E40000000A0406457C0FFFFFF9F5BD83E4000000020616357C0000000803BDC3E40000000E0E46357C00000008033DF3E40FFFFFF3F626457C000000040E0E23E4000000060206457C000000000E5E63E4000000020FA6257C0FFFFFFFFC3E73E40000000002D6357C0FFFFFF3FC5EC3E4000000040EB6157C0FFFFFFFF4DED3E40000000C0A36157C0FFFFFFDF91EF3E40000000600F6257C0FFFFFF7FF2F53E40FFFFFFBF636257C0FFFFFF5FF6F43E40000000801D6357C0010000605EF83E4000000020A36457C0010000C0E6F93E40000000A0E66357C0FFFFFF5FDFFD3E40000000C0886457C0FFFFFF1F4DFF3E40FFFFFF5F586457C0000000004F033F4000000000286457C0FFFFFFBF9F043F4000000000046357C0000000C09E033F4000000040766057C0FFFFFFDFF5093F40FFFFFFBFA56157C000000040960E3F4000000080166157C0000000A01D133F40000000A0D26257C0FFFFFF5F16153F4000000080C26257C0010000A041183F40000000C0D66357C0FFFFFFBFBC193F40000000A0A06357C0FFFFFFDFFD1B3F40000000E03E6257C0000000E0B61D3F4000000040CC6157C0000000A03C203F4000000080666257C0FFFFFF9FE7213F4000000000D46257C0FFFFFF1FBF283F4000000020D06157C0000000A0B6293F40FFFFFF7F5E6257C0000000A0242D3F4000000040B96157C001000060962D3F40000000E0D96157C0000000E08E2F3F40000000E03C6357C0FFFFFFBFE0303F40000000A0EC6457C0FFFFFF5F112C3F4000000020036657C000000080212E3F40FFFFFF3F966657C000000000F6323F4000000080CB6557C0000000C0CC3A3F40000000A01A6757C000000000003E3F4000000000296757C0000000E020453F40FFFFFFFF736757C0000000209B463F40FFFFFF7F5F6857C0000000801E463F40FFFFFF5F516957C0000000A04E4A3F4000000000FE6957C00000004063493F40FFFFFF1F9F6B57C0000000C00B503F40000000A0546B57C00000002011543F4000000080A16857C0FFFFFF1FB35F3F40000000004F6A57C000000040555F3F40000000407F6A57C0000000E0F8653F40000000E0FF6B57C000000020F8673F40000000C0716C57C0000000E0796A3F40000000608D6C57C001000020806D3F40000000E0F76B57C0FFFFFF5F29703F4000000060EC6C57C0000000803D723F40000000E0B26C57C00000002022763F40000000A0836E57C0FFFFFF1FA0753F4000000060147057C0000000C0497C3F4000000020077057C0FFFFFF1F957D3F4000000000046E57C0FFFFFFBFD27E3F40FFFFFFBF2B6D57C0FFFFFFFF43853F40FFFFFF7FD36E57C0000000C099853F40000000C0D76F57C0000000E0A7893F4000000000DA7057C0FFFFFF7FDD873F40000000A0ED7157C0000000E0A2883F40FFFFFF7FDF7357C0FFFFFFBF1E8F3F40000000A03E7457C000000080BD933F40000000A0497557C00100004016973F40000000207A7557C0FFFFFFFF7C9D3F4000000020747457C0FFFFFF3F3B9E3F4000000080247457C000000020E1A53F40000000809C7357C0FFFFFF3F5DA73F4000000040F57357C000000060B0AC3F4000000080B47257C0000000001EB63F40000000E0C27357C00000000023B53F40FFFFFF1F287457C0010000A05CB63F40000000E0D67357C0FFFFFF5FF8BA3F40000000C0317557C000000000D7C03F40000000C09C7457C0000000A04EC63F40000000A0657557C00100000051CD3F4000000040597757C0000000C038D13F4000000060277857C000000000A1D93F40FFFFFF9F667857C00000006015DF3F40000000201F7957C0000000A0BCDE3F40000000808D7957C0FFFFFFFFFAE43F40000000001A7B57C0FFFFFFFF80E43F4000000040BF7A57C0FFFFFF3FE2E83F4000000000E37B57C001000020D2E83F4000000040147E57C00000008054EC3F40000000A08A7E57C0FFFFFF7F37F23F4000000000488057C0000000605AFA3F4000000000A28057C0010000E037FD3F40000000803E8257C00100006098FE3F40FFFFFF9F398257C0000000808719404000000040418257C0FFFFFF1FD2314040000000A0928257C0000000A0EF584040000000A0A98257C0000000C0F0704040000000E07A8257C000000020FB82404000000000D37357C000000060E482404000000060C06057C000000020B5824040FFFFFF3FA65E57C000000000BD82404000000040DF4E57C0000000607682404000000020A53E57C00000004052824040000000A0E42D57C0FFFFFF3F23824040000000400D0457C00000008048814040FFFFFF1F14DD56C0000000A0C6814040000000A05CDB56C0FFFFFFBFB7814040000000A04BD056C0000000A0B9814040FFFFFF5F60CA56C000000040AB814040000000005CCA56C0000000A009804040000000E062CD56C000000000CA77404000000020B1CC56C0000000400375404000000060DCCA56C0000000C0D773404000000080C7C856C0000000007A754040000000E050C856C0000000E0FE7740400000008078C856C000000020D47C40400000000028C756C000000080617E4040FFFFFF5FEEC556C0FFFFFF5F6A7E404000000080D6C456C000000040137A4040FFFFFFFF16C556C0000000E0467040400000000019C956C0000000E0AC6B4040FFFFFF7F01CA56C0000000809F614040000000E0BBC856C0000000E024604040FFFFFFDF40C656C000000040EC5F404000000020C7C356C0000000409C5C404000000080EEC856C0000000E0E2544040000000605CC956C0000000808A524040000000801AC756C0000000C07B4C40400000008015C456C0000000209E4D4040000000600CC356C000000020A34E404000000000C2C156C0000000C01C5240400000002081C056C00000002043524040000000A0C0BF56C0000000002050404000000000EABF56C0000000A0C74E4040000000A011C256C000000000954A404000000080C4C356C000000020C7494040FFFFFF5FB0C456C0000000E0E2474040FFFFFF9F09C456C0000000806545404000000040AFBF56C0000000807D414040000000C07DBF56C000000060653F40400000008042C156C000000000D33E40400000008061C256C0000000409C3F4040000000A0AEC556C0000000C0394640400000008099C756C0000000A0144340400000002088C756C000000020D13F4040000000A0BEC356C00000004037394040000000E0B8C156C000000020A13840400000006032BF56C0000000C0FA39404000000080CABD56C0FFFFFF1F53384040FFFFFF3F14BE56C000000060A2354040FFFFFFDF96C056C0000000A0D332404000000060EDBF56C0FFFFFFBFE02E40400000002003BF56C0000000609E2D4040FFFFFF1F8EBA56C000000060462C404000000060A6B856C000000040D0304040000000C005B856C0FFFFFF5F98304040000000E0A5BA56C0000000A014274040000000C078BB56C00000006011264040000000A02FBE56C0000000A0DF264040FFFFFFDFC4BE56C00000008019264040000000E0F6BE56C0000000C0C02440400000002025BE56C0000000C070224040000000C08ABE56C000000080981C404000000060AAC256C0000000A0ED1F404000000060FFC356C000000080CA1D404000000060A0C556C0000000E0BE1D4040FFFFFFBF6FC656C000000060811B4040000000C06BC756C0000000A0E01C4040000000A010CA56C0000000C0591A4040000000A0ADCA56C0000000C003164040000000C004CA56C0000000408812404000000020F5C656C000000060471140400000000050C556C0000000A0041340400000008075C356C0000000C03311404000000060E6C256C0000000604F144040000000807CC356C0000000C09E1740400000000090C156C000000040BF154040FFFFFFBF3EC056C000000060AA154040000000E0B8C056C0000000E02811404000000020A1C256C000000020C80D404000000060B9C456C000000000080C40400000002029C456C0000000A07E07404000000020B7C556C0FFFFFF9F46074040FFFFFF1F0AC856C0000000A04B0B4040FFFFFF3F51C956C000000060F70A40400000002027C956C00000002076084040000000808FC556C00000008096054040FFFFFF3FAEC456C00000006018044040000000E0A1C456C000000000B202404000000080E9C656C0FFFFFF5FE8FD3F40000000206FCA56C0000000A000FD3F4000000040E4CC56C0FFFFFFDF0EEA3F40FFFFFFBFA3CF56C000000020E6E03F40000000E0EED056C0000000808CDD3F400000000071CF56C0FFFFFF5FD1D53F40000000200AD056C0FFFFFF9F93D13F40FFFFFF3FB5D156C0FFFFFF9FCED33F40000000E0B8D256C0FFFFFF3F80DC3F40000000406AD356C000000080ECDC3F40000000406CD556C0FFFFFF9FE3D73F400000004037D656C001000000B1CB3F40000000407DD756C0000000E04CC53F400000006083D556C00000000072C33F40FFFFFFFFCDD056C0FFFFFF5F15C63F4000000060E3D056C00000002075C23F40000000A014D256C0000000E0EDBF3F4000000040BFD556C0FFFFFF1F25C23F4000000060B4D756C000000000D9C03F40000000805BD756C0FFFFFF5F67C03F4000000040D2D856C0000000C06DB73F40FFFFFF7FE6D856C0FFFFFF5F89A73F400000006021DA56C0000000E017A03F40000000C043DD56C00000000076A03F4000000080D9DF56C00000004096A63F40000000007EE056C0FFFFFFDFC9A43F40000000808FE056C0000000002D9F3F4000000060CFDF56C000000060AC9A3F40000000C093DA56C0000000A0389A3F4000000060DDD956C0FFFFFF5F10963F40000000001ADB56C0FFFFFF3F05903F4000000000B8DD56C000000060EE8A3F40000000802EE056C0FFFFFFFFE2883F400000004068E056C00000008088863F40000000807FE056C000000040F2733F400000008048DE56C0FFFFFF9F926B3F40000000C0CFDD56C00000000066673F4000000020C4DE56C000000020AB603F4000000060A7DF56C0FFFFFF3F05603F400000002077E156C00000000055643F40FFFFFFDF2DE256C0FFFFFFBFB8683F400000004038E256C0FFFFFF7F256F3F40000000C059E356C0FFFFFF7FD46E3F40000000C053E456C000000020D66B3F40000000E0CAE356C0FFFFFFDF57633F400000008034E256C0FFFFFFBFF0613F40000000A0D5E256C000000060635E3F4000000020BBE256C0FFFFFFDFC8583F40000000605CE056C0FFFFFFDFC2523F40FFFFFF9F26E056C0FFFFFFBF764C3F40000000E015E156C0000000006C483F40000000A0A2E356C00000002032453F400000000095E856C0000000800B473F400000002030E956C0000000805C453F40FFFFFF9FC2E856C000000080AD3D3F400000004074E656C0FFFFFF9FB8363F4000000000D9E556C001000080A52D3F4000000000AEE756C0FFFFFFFFA4203F40000000A026E456C000000040FB143F400000006053E356C000000000D80E3F40000000A0A6E456C00000000050083F40000000802BE856C00000002004033F40000000A075E856C0000000C04D003F400000006076CB56C0FFFFFF9F12003F4000000060A4C356C0000000C00F003F400000006018B556C0000000A059003F40000000A0D9A356C00000002064003F4000000040B9A256C0FFFFFFFF8C003F4000000000FD9556C00000008055013F4000000020989056C000000040D7013F40000000C0577556C0000000E0CA013F40000000A0E36E56C0000000E0DB013F40000000A04E6E56C0000000E057003F4000000060826E56C0FFFFFFBF03FC3E4000000060106E56C0000000E0ADF93E4000000020536F56C00000006018F73E40000000E0E66F56C0010000201DF33E40FFFFFFBF626F56C00000006009ED3E40FFFFFFBF827056C000000060BDEB3E4000000040B96F56C0FFFFFF9FF0E83E40FFFFFF5FCA6F56C0FFFFFF5F11E73E40000000E0D87056C0FFFFFF1F79E63E4000000080027156C0FFFFFF5FA5DC3E40000000A03C7256C00000004084DC3E4000000020957156C00000008091D43E4000000000F27256C0FFFFFFFF33D43E40000000C0F97256C0010000C0FDCF3E4000000020E57356C0FFFFFF3FBFCF3E4000000080C67456C0FFFFFF1F1FCA3E40FFFFFFBF0C7456C0000000A07BBF3E40000000E0D47456C0000000E011BE3E40FFFFFF5FB17456C0000000209DBB3E4000000080797556C000000020ABBA3E40000000C0037656C0FFFFFF1F55B63E4000000060297556C0000000A01EB43E4000000060D77556C0FFFFFFDF4AB33E40000000A0DE7556C0000000A0E8AD3E40000000E0757556C001000040FAAC3E40000000A01A7656C0FFFFFF1F83AA3E40000000C0157556C000000000B2AB3E40FFFFFF1F6F7456C0000000A0ABA63E40000000C08C7356C0FFFFFF5F36A63E40000000C07E7456C0000000C0C39F3E40000000209C7256C0000000C0C28D3E4000000060747156C000000000128D3E40000000C0537056C000000040F3833E4000000040DD6E56C0000000C0657F3E4000000000046E56C0FFFFFFBFEF7E3E40FFFFFFBFA36D56C001000080317B3E4000000000856C56C0FFFFFFBF617A3E4000000040736C56C0FFFFFF7FCE773E40000000A0BC6B56C00000002068763E40FFFFFFBF396B56C000000000F6713E4000000040386B56C0000000A05E663E4000000020E56956C0FFFFFFBFFD603E40FFFFFF5F626956C0010000A0E75A3E4000000080896856C0FFFFFFBFE85A3E4000000060B06756C000000040DE573E40FFFFFFFF956756C000000040D5523E4000000000C56856C001000040C84F3E4000000040E76856C0000000C0AE4B3E40000000800E6856C000000000484A3E4000000060C86756C0FFFFFF9FBB413E40000000E0CC6656C0FFFFFFDF643F3E4000000060096756C0000000E0C03D3E40000000A0B96456C000000060DA313E40000000C09F6E56C0FFFFFFFF492E3E4000000000967056C0000000201C3B3E40000000C0617C56C00000004008453E40FFFFFFBFD48456C0FFFFFF9F685E3E40000000C0568F56C00000004079613E40000000C0C59356C000000000BC4D3E40FFFFFF1F2A9B56C0000000A0882F3E40FFFFFF9F4F9956C00000006085173E4000000060A89156C0FFFFFF7FD50F3E4000000060228756C0FFFFFFBF990A3E4000000000647F56C0000000C0AF0D3E40000000C0087956C0010000A0E8273E40000000C0157356C0000000A0EC1A3E40000000A05C6F56C0000000A0A2283E4000000040646E56C0FFFFFF1FF31E3E4000000040D36A56C0FFFFFF7FC6293E40000000C08E6956C0000000604A1F3E40000000C0DF6D56C000000040160E3E4000000060547656C0FFFFFF1FAF023E40000000C0877456C0000000807AF33D40000000E0C16D56C0010000A01DF83D40000000C0936D56C0FFFFFFFFB6E53D40000000A0266856C0010000201FE03D40000000E07D6556C000000040E0E53D40000000C0C16456C0FFFFFFFF3D023E4000000040E45B56C001000000510B3E4000000060075D56C0FFFFFFBF4BFC3D40000000002D5856C0000000407AF33D4000000080955B56C0FFFFFFBFA8F03D40000000E0BC5956C00000008082D83D4000000060F35A56C0FFFFFFDFF0D33D40000000A04B5756C0FFFFFFFFEBCB3D4000000040B65A56C00000002061C83D4000000020DC5E56C0FFFFFF9FABD43D40FFFFFF9F966256C0FFFFFF7F28C13D4000000040A26956C0FFFFFFFF45C43D40000000C0F06556C000000080EDB53D4000000060276756C0000000E091B23D40000000C0C26056C00000008016AA3D4000000020AA5E56C0FFFFFF5FCEA23D40000000E0436656C0010000003AAA3D40000000204E6B56C000000080E7B33D40000000407F6C56C000000060A1B13D4000000020A76856C00000008056A03D40000000C09B6E56C0000000E05DA53D4000000020E76F56C0000000401DA33D40FFFFFFFF476E56C0010000A0169B3D40000000C0637156C0FFFFFF1F2B9C3D4000000040DA6256C0FFFFFF3FB2783D40000000805E6256C0FFFFFF5FB7663D40FFFFFF1F9B5856C000000020D1653D4000000080915556C00000008036573D4000000040F15056C0000000E0B6593D40000000E0CC5056C0000000E02E4C3D40000000E05F4C56C0000000004D593D4000000020504856C000000040694A3D40FFFFFFFFB04756C0010000802F363D4000000040204256C0FFFFFF1F23393D40FFFFFF5F494656C0FFFFFFBFC8293D40FFFFFF7F644156C0FFFFFFDF9A253D4000000080B14356C000000000C6153D4000000080184856C0000000E094223D4000000080204756C0FFFFFFBF31153D40000000C0DA4956C0000000E0960E3D40000000003D4956C00000000036043D40000000606E4F56C0FFFFFF1FF61E3D40000000007A5056C0000000002A0F3D4000000020455956C0000000C07DF03C40000000E0D85056C000000080E2253D40000000206B5456C0FFFFFF1F122E3D4000000060BD5556C000000040B11A3D4000000000DF5856C0FFFFFF5FA3193D4000000040285956C0FFFFFFBF62253D40000000A0E35D56C00000002052373D4000000040615D56C00000006069413D40000000A08F5F56C0FFFFFFFF193C3D4000000060A66756C0FFFFFF9F83473D4000000000156756C0000000A0DF543D4000000040DF7256C00000006083523D40000000A03B7056C0FFFFFFDFC25F3D4000000060967456C000000080B26B3D4000000040517456C000000060377A3D40000000E0DD7D56C000000020F3783D40000000E0327E56C0000000E0D9803D40000000806C8056C000000060647E3D40000000803B8756C0000000E0BA8D3D40FFFFFF1FC88856C0FFFFFF9F99883D40FFFFFFBFB58956C00000000058983D4000000000518D56C000000040658B3D40000000C0238B56C0000000C0E77E3D40000000004E8256C0000000806A723D4000000040898356C000000080956D3D40FFFFFF5FE98156C001000080BD5F3D4000000060938356C0000000C0E8593D40000000801B8256C000000080034F3D40000000E0238756C0010000E04D523D40000000C0FA8456C001000060DF363D40FFFFFF1FC18256C0010000E033393D4000000080F98456C0000000401C2D3D4000000060928E56C0000000A033193D4000000060E39056C001000040362F3D40000000A08A8F56C0000000E027413D40000000E0D69156C00100000062463D40000000602D9256C0FFFFFF7FCC3E3D40FFFFFF7F3E9656C0000000E00E503D40000000C05D9956C0000000C0A9453D4000000020089A56C0FFFFFF7F65533D40000000E0D39C56C0000000002B5A3D40FFFFFF1F7D9E56C001000060C04D3D40000000C01AA756C000000020084E3D40FFFFFF9F50A556C000000060BD423D4000000060C1A756C0FFFFFF5F05393D40FFFFFFFF98A956C0000000800A413D4000000080D5A856C0000000408F293D40000000E0C2AB56C000000060832E3D40000000204FAB56C000000020DA233D400000008019B256C00000004070203D400000004072B156C0000000C00F293D40000000E0B6B556C0000000009F2E3D40000000A07BB856C0FFFFFF7F1E233D4000000080E6BA56C0FFFFFFDF822E3D40FFFFFFDF1BB456C0FFFFFF1F85383D40000000A061B456C000000020AF413D40000000A06CB956C0FFFFFF1F77443D4000000080EEBB56C0FFFFFF5FE2573D40000000A002C556C0010000E01B5C3D40000000A088C656C0000000C05C503D4000000020ACCD56C000000060DC673D40000000C0CED056C0FFFFFF7F477D3D40000000E0B4DB56C000000000688D3D40000000600EE356C0FFFFFF7F0E883D40000000A018E356C0FFFFFFBF54A43D40000000002EE956C000000000CDA43D40000000E066E756C0000000A0D8C43D40000000A03DF756C0000000E0C2B93D400000004063F856C0FFFFFFFF07C43D40000000402DF656C0FFFFFF7FE5CE3D4000000060B9F456C00000002064C93D40000000E0F3F456C0FFFFFFDFBDD63D40000000C0E2FD56C00000006078D73D400000000049FE56C0FFFFFF1F4CCE3D4000000000C50857C00000000004BB3D4000000080510857C000000000F8C53D40FFFFFFBFC50C57C0000000004EC33D4000000000D10357C0FFFFFF3F579B3D4000000000061357C000000000968A3D40000000C0DD2657C0000000A0A1963D40000000C0F14E57C000000000EDC93D4000000080326E57C0010000E031C23D40000000604E7357C0FFFFFFDFC3B93D4000000040957957C0000000C04CCF3D4000000080A47257C000000080ADD93D4000000020A77057C0FFFFFFDF86013E4000000040996D57C0000000407E0F3E40000000A0D06D57C0000000C07D183E4000000020566D57C0000000A05F1D3E40000000609A6C57C0FFFFFF9F301E3E4000000080B66C57C00000006027243E40000000C0E66B57C0FFFFFF1F29243E40000000A0B86B57C0FFFFFF7FF2253E40FFFFFF3FC76C57C0FFFFFF5F9B263E40000000008E6C57C000000020F92C3E4000000040146D57C0000000E04C2E3E4000000000C06D57C00000000066383E4000000060456D57C000000080473D3E4001030000000100000008000000FFFFFF1F0A0157C0000000C0A4983D4000000060C1F956C0FFFFFF7F95A63D400000004043F156C00100002012943D4000000060E5EC56C0FFFFFFBFB9933D4000000040F3F056C0000000C0B6883D400000000083F056C0010000408A7E3D400000008055F656C0FFFFFF3FA37C3D40FFFFFF1F0A0157C0000000C0A4983D400103000000010000000F00000000000060D6D556C00000002079573D400000008039D356C0FFFFFFFFF4503D40000000207ECE56C0FFFFFF1F95613D40000000601BCC56C0FFFFFFDF214C3D400000006055CA56C000000000D1523D40000000A0F3CA56C0000000A090483D40000000E006CC56C000000080D7483D40000000A0D2CC56C0000000C0C94E3D40000000207ACC56C00000000001463D4000000060C2C956C0FFFFFF1F2D443D40000000C077CA56C000000060BB3E3D40FFFFFF1F9DC856C0000000007C423D400000008034C856C0000000200E3A3D40000000E0AAD156C0FFFFFF7FF9403D4000000060D6D556C00000002079573D400103000000010000001200000000000080CEBB56C0FFFFFF7F45423D40000000C020BE56C0FFFFFF7FEB3D3D40000000A006BD56C0000000A0D8453D4000000060CDBE56C000000060B0463D4000000000BDBE56C00000004040383D400000000079BC56C000000040E6393D400000004090BD56C0000000E0812F3D40FFFFFF7F33C056C0000000C0202F3D4000000060A5BF56C0FFFFFF1F7E393D40000000A0D6C256C00000008006363D40000000C01CC256C0FFFFFF3F81463D400000006053C456C001000000BE403D40000000A08CC356C000000040D1303D4000000060D6C756C0010000C0133A3D400000008027C856C0FFFFFF5F154B3D400000002077C056C0FFFFFF7F1D4C3D4000000040EDBF56C000000040D6523D4000000080CEBB56C0FFFFFF7F45423D40 Louisiana 22 0.35937500000000000000 0.34299516908212560386 0.36095346197502837684 0.35650887573964497041 0.36261980830670926518 0.38970588235294117647 0.40166204986149584488 0.38018433179723502304 0.37197049525816649104 0.43773584905660377358 0.40044493882091212458 0.35443037974683544304 0.38573883161512027491 0.37997432605905006418 0.49340866290018832392 0.54784240150093808630 0.54075425790754257908 0.47410358565737051793 0.50000000000000000000 0.56112334801762114537 0.59501385041551246537 0.53831325301204819277 0.54726368159203980100 0.53316645807259073842 0.54883530854107069881 0.55445134575569358178 0.55282944202611792639 0.54638050200072753729 0.57508896797153024911 0.59868180153789820579 0.59498587570621468927 0.57758031442241968558 0.57232084155161078238 0.56721105527638190955 0.58738461538461538462 0.58894442810937959424 0.59698576611777839799 0.60170366546205472380 0.60238379022646007151 0.61872608597794283142 0.59851454507943057561 0.61021611001964636542 0.62792452830188679245 0.63068281551693873969 0.63996154462425893286 0.66196976368706883898 0.68462494819726481558 0.70475586556753329106 0.70420110192837465565 0.71073619631901840491 0.71215021420107556285 0.71010531393198810194 0.72390912369275153264 0.70844796349728242636 0.68770175327552376733 0.66143344709897610922 0.64600543623087992325 0.60025950693682004192 0.56021922813427723224 0.54621568709126504769 0.54258247082994146606 0.56938210652304009575 0.59844395637121691546 0.59256155055002619172 0.59850685764475373286 0.61505960958993842526 0.61166932732337934704 0.60510277246653919694 0.59582537363748735813 0.59681725942539784257 0.58134860050890585242 0.57416338582677165354 0.59405378556730291634 0.61119440279860069965 0.62347280532100570782 0.61712102024039636517 0.63342690655437880141 0.65394168149727182055 0.64872983497125903950 0.65525968309859154930 0.66654657160194174757 +44 0106000020E6100000030000000103000000010000006B010000000000A03CC953C000000040E545424000000060E4CD53C0000000405F464240FFFFFFBFA8E053C0FFFFFF9F19464240000000A0EAED53C00000004021464240000000208A0154C000000060C345424000000000140354C00000008008464240FFFFFF1FDC1B54C0000000A088464240000000801B2754C0FFFFFF7F5547424000000060A43554C0000000C01E48424000000000D23954C0000000E05848424000000060195654C0FFFFFF9F5349424000000040E16A54C0000000A0794B4240FFFFFF7FC16954C000000060C44D4240000000400F7554C000000000454E4240FFFFFFDFC77A54C000000000874E4240000000407C7B54C000000060444C424000000080E08954C0000000602A4C424000000020E08D54C000000020074C424000000000029354C0000000C0BC4B4240000000001AA754C000000080B44B4240FFFFFF5F65B654C000000020A44B4240000000E027BF54C000000060AC4B42400000008081CD54C0FFFFFF9F434B424000000040E7CF54C000000020804B4240FFFFFFBF9BD154C000000040D94C424000000020B7DD54C000000000A44C42400000008037EB54C0000000A09F4C4240000000A066E954C000000060F64E4240000000A0FBE154C0000000A0A8544240000000C075DD54C000000020B454424000000020DFD954C0000000000C56424000000060B3D854C0000000C014584240FFFFFF1F93D454C000000020CF5A4240000000600ACD54C000000060F95D424000000020DFC854C000000060B75E4240000000C0F7C754C000000060236042400000008036C854C000000060B8634240000000205BC454C000000080EA6C4240000000E0FDC254C000000000EA6D4240000000E0DBBC54C0FFFFFF1F976E4240000000C033B854C000000080617242400000008016B754C0000000004E774240FFFFFF9F77B754C000000020BC7C424000000060FDB354C0FFFFFF5FB48042400000006051AE54C0000000A0568442400000006017AE54C0000000206D8842400000000065AD54C000000020A6894240000000002DAE54C000000000E88B4240000000A007AE54C0000000C0118E4240000000805CA454C0000000E0CE984240FFFFFFDF35A354C0000000C08199424000000020FC9954C00000008013A04240FFFFFF7FA79654C00000002055A1424000000060809254C00000002002A74240000000406C7D54C0000000E0F9C3424000000000E17B54C000000020D6C0424000000080AE7C54C0FFFFFFDF17BF424000000040807E54C000000040CCBD4240FFFFFF3F417F54C000000080B5BB4240000000E0EF7A54C0000000002CB54240000000A0537B54C00000002091AF4240000000A06D7954C0000000C094AB4240000000604B7754C0000000E0A4A9424000000040F77654C00000002049A74240FFFFFFFFB27554C0000000C087A44240000000E0317454C000000040C4A34240000000A0BD7254C0000000C0BDA4424000000020217054C0000000A0D5A24240000000A0456F54C0FFFFFF5F0CA04240000000E0EB6C54C0000000001F9E4240FFFFFF9FA06A54C0000000C0369A424000000000A36354C000000000669A4240000000805D6054C000000020FC9D4240FFFFFF9FB95F54C0000000A059A04240000000206F5E54C0000000208DA0424000000040D35954C00000004029A4424000000020085954C000000020D0A7424000000060F95654C0000000005FAB4240000000A0F85354C00000006094A5424000000080474E54C000000080BB9E4240000000E0044954C0000000E02CA34240000000A09B4154C0000000C099A44240000000C01C3F54C0000000002FA7424000000080A33E54C000000000EFA5424000000020F53D54C00000008055A54240000000E0CC3B54C0000000608FA64240000000A0C23654C0000000E06DAB4240000000C04F3654C0000000C0E7AC4240000000E0293854C0000000E0BCB1424000000040723654C00000000030B6424000000020303354C000000040DBB44240000000202A3354C00000000021B24240000000604B3154C0FFFFFFDF6AB1424000000040D83054C0FFFFFF7F86AF424000000080DD2F54C0000000C081B0424000000020C72F54C0000000609DB14240FFFFFF5FB72E54C0000000A040B2424000000080252D54C000000060B2B14240000000C0402654C00000004014B9424000000000C02254C0FFFFFFFF0ABC424000000020932054C000000040CABC4240000000003D1F54C0000000C0F0BA4240FFFFFF3F2B1F54C0000000A084B7424000000020661E54C0FFFFFFDF1AB64240000000003D1B54C0000000E0A6B74240000000A0DD1854C0000000C098BB4240000000608D1654C0000000C0ECBC424000000040411654C0000000C0DABE4240000000E0701254C00000004069C1424000000020FB1154C0FFFFFFFF9FC44240000000E0BE1354C0000000809DC3424000000040DF1454C00000006042C44240000000C0481454C00000002086C8424000000080C80F54C00000000063CC424000000080060E54C0000000E0E3CF4240000000004C1054C000000080FED1424000000000461354C000000060F9D1424000000060861354C0000000C078D3424000000040F01254C0000000C0EFD54240000000A0691354C0000000C05DD74240FFFFFFFF031054C000000040EBDC424000000040501054C0FFFFFFFFE8E04240000000E0200E54C0FFFFFF9FADE3424000000020550E54C0FFFFFFDFAFE64240000000E0FE0A54C0FFFFFF5FE2EB424000000020090B54C00000008016EE4240FFFFFFFF400A54C00000000045F0424000000020990754C00000006011F24240FFFFFF3FD40654C0000000800FF5424000000080850354C0000000A04EFA4240000000C00B0054C000000000B0FE4240000000A0DEFD53C0FFFFFF7FED044340000000804BFD53C0000000609B084340000000C06CFB53C000000040350D434000000000E0FB53C000000000830F4340000000A046FA53C000000040CC14434000000000A6FA53C000000020EE1643400000004035F553C0000000200520434000000060CEF253C0000000A05F224340000000E059F253C0FFFFFFBF7A2443400000006064F353C0000000603D264340000000403CF353C0FFFFFFDF3728434000000020E9F053C0000000A04B2D434000000060EAEE53C000000020052D4340FFFFFFBF18EE53C00000002081324340000000E0CBEB53C0000000200E3743400000002058EC53C0000000C007404340000000209DEA53C0FFFFFFFFA4424340000000A0DBEA53C00000006068464340000000E020E953C000000080CE4B4340FFFFFFFF59E253C0FFFFFF3FDF4643400000002024DF53C0000000A0233B4340000000A04DD453C000000020CD3443400000004072D153C0000000A0F537434000000080D7CE53C0000000E07C3D4340000000C02BC853C0000000803D54434000000080C3C753C0FFFFFF7FF2544340000000C0AEC553C0000000005D5443400000004099C553C000000000845A4340000000A0A2C353C0FFFFFF3F87614340000000E085C353C0000000C02F654340000000E02CC253C000000060616643400000008036BF53C0000000005F6C4340FFFFFFDF79B753C000000080B3614340000000403BB453C0FFFFFF7FB46A4340FFFFFF9FC5B253C000000060A770434000000000F8AF53C000000020A8744340FFFFFF3F3BAF53C000000020EF764340000000A05CAE53C000000040117743400000002008AE53C000000020D3734340000000208DAB53C0FFFFFF1FF6754340000000406CA953C000000020A87943400000002064A853C0000000A0637D43400000006055A653C000000020CD7B4340000000206CA353C000000020C5814340000000802AA353C0FFFFFF1FFE824340000000E01FA453C0000000207C844340000000E05CA253C0000000A04C874340000000A01EA053C000000060FA8B4340000000C0129F53C0000000C0508E434000000020B09C53C000000020398F4340FFFFFFDF929B53C0000000C002934340000000C0C49953C0000000A0D2954340FFFFFF5F289B53C0000000804899434000000000189B53C000000020249B4340000000C08F9953C000000040579F4340FFFFFFFF7B9A53C0000000C0F3A04340000000E0D49553C0FFFFFF9FB1AB434000000060079653C0FFFFFFDFE8AC434000000060689753C00000008048AE4340000000A06E9653C0000000C0BBB04340000000A0429653C0000000C07BBA4340000000E0BC9153C0000000E030B64340000000C0B48E53C0000000C00CB24340000000A0268253C000000020FDA14340000000802E7553C0000000E0E790434000000040807453C0000000A02092434000000020917353C00000000027994340000000602F7153C0000000A08B9F4340000000A09F7053C0FFFFFFDF6BA44340FFFFFF7F936E53C000000040AAA84340000000607E6B53C000000080CAA8434000000020756753C0000000205DA64340000000E0696453C0000000C031A64340FFFFFF3FB36253C0000000A06CA24340000000C09E5F53C000000020FD9F4340000000E0C15D53C0000000C0519D4340FFFFFF5F915D53C000000020FC9B434000000000A25E53C0000000C0A596434000000040106153C00000002027944340000000C0D56053C000000080EE8E434000000000AC5E53C0000000804E8D4340FFFFFF9F6B5D53C0000000E0588A4340000000E0B65B53C0000000208C884340000000602D5653C0FFFFFFFFC488434000000040C65453C0000000E00288434000000020625053C00000008087834340FFFFFF3F994F53C0000000A0E97C4340FFFFFF1FBB4953C0FFFFFFDF7D7B434000000020D94753C0000000C04D774340000000800D4553C0000000603275434000000040584453C0000000C06B71434000000040804253C000000020646E434000000000964253C000000020516B434000000060E44253C0000000A0286A434000000080414253C0000000602E684340000000A0E84253C000000020E164434000000060F94253C000000000015C4340000000E0A74353C0FFFFFF7F235B4340FFFFFF7F3D4553C0000000408E5B434000000020F64553C0FFFFFF1F1C5A4340000000E0014853C000000020C2564340000000C0514853C0000000C0F5524340000000E09F4C53C0000000A0B44F4340000000C0764C53C0FFFFFFFF93544340FFFFFFDF904E53C0000000E04A534340000000806E5353C0000000A03E404340000000C0A95553C0000000E0E9374340000000E0865253C0000000006C2E4340000000C0985453C0000000A0072C4340000000A0674F53C0000000606A2A4340000000807D4353C0000000800B30434000000080F53F53C0FFFFFFFFDF23434000000000EF3B53C0000000A0EA194340000000601E2653C000000080630F4340FFFFFF5F232353C0000000807C09434000000000B72353C0FFFFFFFF3D034340FFFFFF5FB72453C0000000E06700434000000020922153C0000000A0A101434000000020891753C0000000607DFA4240FFFFFFBF961053C000000080ECF14240000000801F1053C0000000A0D2EC424000000040C51453C0000000803FE6424000000020D61353C0FFFFFFDF0BDC424000000020D91653C0000000E09DD9424000000000AD1453C0000000A0C2D6424000000060121653C000000000BCCF424000000020742053C00000006004D4424000000040262553C00000002093E24240000000006F2853C0000000C0EEE5424000000000663153C0000000A055F5424000000060623453C000000060B2F54240000000E0DE2E53C0000000A034E6424000000080A12B53C0000000202BE34240FFFFFF9F722453C0000000002AD2424000000020231453C0000000C08DC6424000000020501653C00000002038C34240000000A0D22053C0000000C0BAC64240000000A0C91B53C000000020F2C1424000000040C31653C00000004004C24240000000404B1053C0000000C0F1B14240000000C09C1153C00000000048AA424000000020431353C000000040D3AA424000000080B21553C0000000C05FB2424000000080991C53C0000000A09EBA4240FFFFFF1FB11D53C0000000409EB5424000000080B11A53C0000000E0C0B4424000000020D71953C000000060BFAF4240000000C0271D53C0000000A051B04240000000A0221953C0000000008FA5424000000040831D53C000000000B2A04240000000C0D22953C000000000C3B4424000000080192D53C00000002091B5424000000000E12A53C0FFFFFF1F92AF424000000020152653C00000004049A54240FFFFFFBF2D1B53C0000000C0889A4240000000806E1A53C0000000A08193424000000060661953C0000000E02596424000000040481753C000000020BE924240000000A0961553C000000040A896424000000080481253C0000000E0A08F424000000000531953C000000060C98D424000000000DA1153C0000000208489424000000020C61253C0000000609F824240000000809C1853C0000000E0C57E424000000080451B53C0000000A08D7B4240000000E0FD2153C000000080A888424000000020FA2053C0000000A04F8B4240000000A0202453C000000020138F4240FFFFFF5F612453C000000020438A4240000000A0FE2753C000000060EE904240000000C00A2753C0000000A0DB964240000000607A2953C0FFFFFF3FE89C4240000000209E2C53C000000040C39D424000000020C02F53C0FFFFFFFFC098424000000080F03253C000000000C59E4240000000E0DB3653C0FFFFFF1F389F4240FFFFFFFF073853C00000004056A9424000000020383853C0000000E034A1424000000060413C53C000000040499E4240000000C0A73953C000000020BC99424000000080083353C0000000E0889A424000000060AB2E53C00000002049934240000000E0E62B53C0000000A05799424000000080FA2A53C000000040E8924240000000E0992A53C0000000E0ED86424000000020FB2453C0000000A02283424000000080412753C0FFFFFFFF567F4240000000E0852353C000000000CB80424000000020541F53C0000000C0197B424000000060192153C0FFFFFF7FC274424000000000DC1E53C0000000A0A4754240000000C0241F53C000000000A272424000000080DF2353C000000020C06B424000000060F52353C0000000C0D6654240000000E0752053C0000000E04A6F4240000000A04A1A53C0000000806173424000000000481653C000000060E874424000000000E21553C0000000A01A6E4240000000C0381953C0000000A0FF6A424000000000AD1953C0000000E0BE694240000000E0501453C0000000A0446C4240000000A0BB1253C000000020076A424000000000B01353C00000008093784240FFFFFFBF301253C0000000E03A7B4240FFFFFFFFF20C53C000000060B077424000000040440C53C0000000C0C473424000000060940753C0000000403F774240FFFFFFFFB3FF52C00000004029764240000000E033F852C0000000E02647424000000020BAF952C000000080314742400000008024F952C0000000C0AC4C4240000000E0D9FC52C0000000405C5C424000000020EAFF52C0000000604047424000000020BD0153C0000000804247424000000080F50353C000000080424D424000000000F10253C0000000404A47424000000040270853C00000002051474240000000E0221553C0000000E02C47424000000000D81F53C0000000E024474240000000C0112453C0FFFFFF7F1247424000000000FC3A53C0000000A0EE46424000000000253B53C000000040EE464240000000C0594B53C00000006034474240000000C07B5453C0000000C0E646424000000040E47053C000000020D7464240000000E0867953C0000000E0C6464240FFFFFF7F4E8353C000000080B7464240000000608F9453C000000000D4454240000000205D9D53C0FFFFFF3F4F454240FFFFFF5F31AF53C0000000C0E545424000000020FDB253C00000008092454240000000A03CC953C000000040E5454240010300000001000000090000000000008053D152C000000000880343400000008086CF52C0000000C0A6034340FFFFFF7F20D352C0000000803FFB424000000020B5D552C0000000A0C3F1424000000080B5D852C00000006015F042400000006011D652C0000000E071F34240000000803AD852C00000002053F342400000002030D652C0000000209BF542400000008053D152C00000000088034340010300000001000000220000000000000083F752C0000000E0ADC64240000000003BFC52C000000000E1C74240000000E07DFB52C000000040FECA424000000020C9F852C0000000C048CA4240000000C0FBF952C000000060CCCB4240000000A02EF352C0000000001CDB42400000002016F252C00000004019E54240FFFFFF9F8CEC52C0000000C089E9424000000000F3EB52C000000000D7ED424000000080F9EE52C0000000E01CF742400000000024EA52C0000000A078F84240000000607CE952C0000000402DFC4240FFFFFF7F17E852C0000000A08AFF4240000000A0DBD752C0FFFFFF9F23024340000000008CE752C0000000A03BD9424000000000C1E552C000000040AED6424000000080C4EC52C00000002075CB4240000000009EE952C000000000A6C74240000000C08FEE52C00000008072C74240000000806AF052C00000006059C142400000002023ED52C0000000202ABF4240FFFFFF9F08F452C0000000A009BC4240000000E082F452C0000000E08DB64240000000009CF252C0000000E03CB4424000000040E8F452C0000000E085B54240000000406AF952C0000000C006AF4240000000E09BFB52C0FFFFFF7F3D924240000000A024FE52C00000006028904240FFFFFF9F2E0153C00000002086A74240FFFFFFFFCDFB52C0FFFFFFBF08BE4240000000E0C9FD52C0000000605BBD4240000000E019FD52C000000060CBC242400000008091FB52C00000002048C742400000000083F752C0000000E0ADC64240 Virginia 51 0.37673611111111111111 0.37101449275362318841 0.41997729852440408627 0.42011834319526627219 0.45527156549520766773 0.47058823529411764706 0.48476454293628808864 0.44930875576036866359 0.44573234984193888303 0.49056603773584905660 0.47385984427141268076 0.45472249269717624148 0.50000000000000000000 0.50385109114249037227 0.52919020715630885122 0.56285178236397748593 0.57785888077858880779 0.57029026750142287991 0.57344632768361581921 0.63215859030837004405 0.62714681440443213296 0.60578313253012048193 0.64224332881049298960 0.62995410930329578640 0.62648140580302411116 0.64347826086956521739 0.64780371982588049070 0.62277191706074936340 0.61850533807829181495 0.65324057121933357744 0.66560734463276836158 0.66165413533834586466 0.65910585141354372124 0.66614321608040201005 0.68246153846153846154 0.70655689503087327257 0.71532235556795981021 0.70882808466701084151 0.70727056019070321812 0.73193787981093855503 0.73406230658139055086 0.74557956777996070727 0.77207547169811320755 0.78743198174477795331 0.79666720076910751482 0.80493174812857771907 0.81972648155822627435 0.82866201648700063412 0.82552800734618916437 0.82208588957055214724 0.81988879773949503236 0.81807219229841627140 0.81435268662098809953 0.81023954908407703147 0.81878599911386796633 0.81331058020477815700 0.81468848265202792730 0.81031041022058089630 0.79159625485270609728 0.77294122547590286167 0.76675582432065744079 0.76817773788150807899 0.78516919182518706027 0.76706827309236947791 0.76930612796432673468 0.77652954277479365911 0.76551788900366231571 0.76168140535372848948 0.75199460613552084504 0.75678201431165224821 0.75798982188295165394 0.76678149606299212598 0.77454528252796896805 0.79091406677613574165 0.81256191329779706590 0.80617209891355062321 0.81459742248309302029 0.79486828466939158665 0.78816984980530317078 0.79608641431924882629 0.81938334344660194175 +\. + + +-- +-- Name: markov_usjoin_example_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY markov_usjoin_example + ADD CONSTRAINT markov_usjoin_example_pkey PRIMARY KEY (cartodb_id); + + +-- +-- PostgreSQL database dump complete +-- + From 14d50facda0994263dcf5ac793e266cbc36b3583 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 3 Jun 2016 11:06:29 -0400 Subject: [PATCH 063/183] adds test that expects all null values returned --- src/pg/test/sql/05_markov_test.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 7b876e3..1abc0dc 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -13,3 +13,10 @@ SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 ON m1.cartodb_id = m2.rowid ORDER BY m1.cartodb_id; + + + SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility + FROM markov_usjoin_example As m1 + JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example LIMIT 0', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 + ON m1.cartodb_id = m2.rowid + ORDER BY m1.cartodb_id; From e7de471ac8065b39463606c149aeddab45dd03ae Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 3 Jun 2016 11:06:48 -0400 Subject: [PATCH 064/183] adds expectation values for tests --- src/pg/test/expected/05_markov_test.out | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/pg/test/expected/05_markov_test.out diff --git a/src/pg/test/expected/05_markov_test.out b/src/pg/test/expected/05_markov_test.out new file mode 100644 index 0000000..fd69a67 --- /dev/null +++ b/src/pg/test/expected/05_markov_test.out @@ -0,0 +1,57 @@ +SET client_min_messages TO WARNING; +\set ECHO none +_cdb_random_seeds + +(1 row) +cartodb_id|trend|trend_up|trend_down|volatility +1|0.0714285714286|0.0666666666667|0.0|0.367574633389 +2|0.222222222222|0.181818181818|0.0|0.317010832258 +3|0.0526315789474|0.05|0.0|0.37549966711 +4|0.1|0.235294117647|0.176470588235|0.215650620939 +5|0.0|0.08|0.08|0.32199378876 +6|-0.056338028169|0.0|0.0533333333333|0.373904325255 +7|0.121212121212|0.108108108108|0.0|0.348470518082 +8|0.0|0.0857142857143|0.0857142857143|0.316614746907 +9|-0.0416666666667|0.0967741935484|0.129032258065|0.291681407883 +10|0.222222222222|0.181818181818|0.0|0.317010832258 +11|-0.294117647059|0.0|0.227272727273|0.299586491878 +12|-0.2|0.0|0.166666666667|0.323178657161 +13|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 +14|-0.12|0.0333333333333|0.133333333333|0.320416395752 +15|0.0526315789474|0.05|0.0|0.37549966711 +16|0.0|0.0857142857143|0.0857142857143|0.316614746907 +17|0.0|0.0|0.0|0.4 +18|0.0|0.0|0.0|0.4 +19|-0.056338028169|0.0|0.0533333333333|0.373904325255 +20|0.238095238095|0.214285714286|0.0357142857143|0.286249498707 +21|0.0|0.08|0.08|0.32199378876 +22|0.0714285714286|0.0666666666667|0.0|0.367574633389 +23|-0.04|0.138888888889|0.166666666667|0.256640210277 +24|-0.04|0.138888888889|0.166666666667|0.256640210277 +25|0.0|0.08|0.08|0.32199378876 +26|0.1|0.235294117647|0.176470588235|0.215650620939 +27|-0.056338028169|0.0|0.0533333333333|0.373904325255 +28|-0.056338028169|0.0|0.0533333333333|0.373904325255 +29|0.0526315789474|0.05|0.0|0.37549966711 +30|-0.056338028169|0.0|0.0533333333333|0.373904325255 +31|0.238095238095|0.214285714286|0.0357142857143|0.286249498707 +32|-0.235294117647|0.0434782608696|0.217391304348|0.281234115283 +33|-0.2|0.0|0.166666666667|0.323178657161 +34|-0.2|0.0|0.166666666667|0.323178657161 +35|-0.12|0.0333333333333|0.133333333333|0.320416395752 +36|0.121212121212|0.108108108108|0.0|0.348470518082 +37|-0.056338028169|0.0|0.0533333333333|0.373904325255 +38|0.0526315789474|0.05|0.0|0.37549966711 +39|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 +40|-0.0416666666667|0.0967741935484|0.129032258065|0.291681407883 +41|0.0|0.0857142857143|0.0857142857143|0.316614746907 +42|0.222222222222|0.181818181818|0.0|0.317010832258 +43|0.0|0.0|0.0|0.4 +44|0.0|0.0|0.0|0.4 +45|0.0|0.166666666667|0.166666666667|0.244948974278 +46|0.1|0.0909090909091|0.0|0.356289417132 +47|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 +48|-0.333333333333|0.0|0.25|0.291547594742 +(48 rows) +cartodb_id|trend|trend_up|trend_down|volatility +(0 rows) From 1f73be2752cb5641caaca1bd3edff5acc21786b7 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 6 Jun 2016 08:46:13 -0400 Subject: [PATCH 065/183] rename functions -> trend --- src/pg/sql/11_markov.sql | 2 +- src/pg/test/sql/05_markov_test.sql | 4 ++-- src/py/crankshaft/crankshaft/space_time_dynamics/markov.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 18d28b6..af5f8c8 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -10,7 +10,7 @@ -- Array['date_1', 'date_2', 'date_3']) CREATE OR REPLACE FUNCTION - CDB_SpatialMarkov ( + CDB_SpatialMarkovTrend ( subquery TEXT, time_cols TEXT[], num_classes INT DEFAULT 7, diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 1abc0dc..24581f9 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,13 +10,13 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility FROM markov_usjoin_example As m1 - JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 + JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 ON m1.cartodb_id = m2.rowid ORDER BY m1.cartodb_id; SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility FROM markov_usjoin_example As m1 - JOIN cdb_crankshaft.CDB_SpatialMarkov('SELECT * FROM markov_usjoin_example LIMIT 0', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 + JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example LIMIT 0', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 ON m1.cartodb_id = m2.rowid ORDER BY m1.cartodb_id; diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index a04f6c0..d60d3ea 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -19,6 +19,7 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, Inputs: @param subquery string: e.g., SELECT the_geom, cartodb_id, interesting_time_column FROM table_name @param time_cols list of strings: list of strings of column names + @param num_classes (optional): number of classes to break distribution of values into. Currently uses quantile bins. @param w_type string (optional): weight type ('knn' or 'queen') @param num_ngbrs int (optional): number of neighbors (if knn type) @param permutations int (optional): number of permutations for test stats @@ -30,7 +31,6 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, @param trend_down float: probablity that a geom will move to a lower class @param trend float: (trend_up - trend_down) / trend_static @param volatility float: a measure of the volatility based on probability stddev(prob array) - @param """ if len(time_cols) < 2: From d41e28bc6fa9f04d919ea7bc177445477c381eb3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 6 Jun 2016 09:26:52 -0400 Subject: [PATCH 066/183] pylint corrections --- src/py/crankshaft/crankshaft/__init__.py | 6 ++-- .../crankshaft/clustering/__init__.py | 3 +- .../crankshaft/pysal_utils/__init__.py | 3 +- .../crankshaft/pysal_utils/pysal_utils.py | 6 ++-- src/py/crankshaft/crankshaft/random_seeds.py | 1 + .../space_time_dynamics/__init__.py | 3 +- .../crankshaft/space_time_dynamics/markov.py | 36 +++++++++++-------- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/py/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py index d07e330..dafb2b9 100644 --- a/src/py/crankshaft/crankshaft/__init__.py +++ b/src/py/crankshaft/crankshaft/__init__.py @@ -1,2 +1,4 @@ -import random_seeds -import clustering +"""Import all modules""" +import crankshaft.random_seeds +import crankshaft.clustering +import crankshaft.space_time_dynamics diff --git a/src/py/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py index 0df080f..e4610e5 100644 --- a/src/py/crankshaft/crankshaft/clustering/__init__.py +++ b/src/py/crankshaft/crankshaft/clustering/__init__.py @@ -1 +1,2 @@ -from moran import * +"""Import all functions from moran clustering""" +from crankshaft.clustering.moran import * diff --git a/src/py/crankshaft/crankshaft/pysal_utils/__init__.py b/src/py/crankshaft/crankshaft/pysal_utils/__init__.py index 835880d..fdf073b 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/__init__.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/__init__.py @@ -1 +1,2 @@ -from pysal_utils import * +"""Import all functions for pysal_utils""" +from crankshaft.pysal_utils.pysal_utils import * diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index c21a834..4622925 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -1,5 +1,6 @@ """ - Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects + Utilities module for generic PySAL functionality, mainly centered on + translating queries into numpy arrays or PySAL weights objects """ import numpy as np @@ -78,7 +79,8 @@ def query_attr_where(params): {'subquery': ..., 'time_cols': ['time1', 'time2', 'time3'], 'etc': ...} - Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT NULL AND idx_replace."time3" IS NOT NULL' + Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT + NULL AND idx_replace."time3" IS NOT NULL' """ attr_string = [] template = "idx_replace.\"%s\" IS NOT NULL" diff --git a/src/py/crankshaft/crankshaft/random_seeds.py b/src/py/crankshaft/crankshaft/random_seeds.py index b7c8eed..31958cb 100644 --- a/src/py/crankshaft/crankshaft/random_seeds.py +++ b/src/py/crankshaft/crankshaft/random_seeds.py @@ -1,3 +1,4 @@ +"""Random seed generator used for non-deterministic functions in crankshaft""" import random import numpy diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py index f6be2b2..a9810a7 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -1 +1,2 @@ -from markov import * +"""Import all functions from clustering libraries.""" +from crankshaft.space_time_dynamics.markov import * diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py index d60d3ea..bbf524d 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -8,29 +8,37 @@ import pysal as ps import plpy import crankshaft.pysal_utils as pu -def spatial_markov_trend(subquery, time_cols, num_classes = 7, - w_type = 'knn', num_ngbrs = 5, permutations = 0, - geom_col = 'the_geom', id_col = 'cartodb_id'): +def spatial_markov_trend(subquery, time_cols, num_classes=7, + w_type='knn', num_ngbrs=5, permutations=0, + geom_col='the_geom', id_col='cartodb_id'): """ Predict the trends of a unit based on: 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) 2. average class of its neighbors Inputs: - @param subquery string: e.g., SELECT the_geom, cartodb_id, interesting_time_column FROM table_name + @param subquery string: e.g., SELECT the_geom, cartodb_id, + interesting_time_column FROM table_name @param time_cols list of strings: list of strings of column names - @param num_classes (optional): number of classes to break distribution of values into. Currently uses quantile bins. + @param num_classes (optional): number of classes to break distribution + of values into. Currently uses quantile bins. @param w_type string (optional): weight type ('knn' or 'queen') @param num_ngbrs int (optional): number of neighbors (if knn type) - @param permutations int (optional): number of permutations for test stats - @param geom_col string (optional): name of column which contains the geometries - @param id_col string (optional): name of column which has the ids of the table + @param permutations int (optional): number of permutations for test + stats + @param geom_col string (optional): name of column which contains the + geometries + @param id_col string (optional): name of column which has the ids of + the table Outputs: - @param trend_up float: probablity that a geom will move to a higher class - @param trend_down float: probablity that a geom will move to a lower class + @param trend_up float: probablity that a geom will move to a higher + class + @param trend_down float: probablity that a geom will move to a lower + class @param trend float: (trend_up - trend_down) / trend_static - @param volatility float: a measure of the volatility based on probability stddev(prob array) + @param volatility float: a measure of the volatility based on + probability stddev(prob array) """ if len(time_cols) < 2: @@ -49,7 +57,7 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, if len(query_result) == 0: return zip([None], [None], [None], [None], [None]) except plpy.SPIError, err: - plpy.debug('Query failed with exception %s: %s' % (err, query)) + plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals))) plpy.error('Query failed, check the input parameters') return zip([None], [None], [None], [None], [None]) @@ -72,8 +80,8 @@ def spatial_markov_trend(subquery, time_cols, num_classes = 7, ## get lag classes lag_classes = ps.Quantiles( - ps.lag_spatial(weights, t_data[:, -1]), - k=num_classes).yb + ps.lag_spatial(weights, t_data[:, -1]), + k=num_classes).yb ## look up probablity distribution for each unit according to class and lag class prob_dist = get_prob_dist(sp_markov_result.P, From 4e86965f033f14252ded35534bbd2c911e104da7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 7 Jun 2016 19:58:32 +0000 Subject: [PATCH 067/183] KMeans clustering and weighted centroid analysis --- doc/11_kmeans.md | 62 +++++++++++++++++++ src/pg/sql/11_kmeans.sql | 31 ++++++++++ src/pg/test/expected/05_kmeans_test.out | 10 +++ src/pg/test/sql/05_kmeans_test.sql | 6 ++ .../crankshaft/clustering/__init__.py | 1 + .../crankshaft/clustering/kmeans.py | 17 +++++ src/py/crankshaft/test/fixtures/kmeans.json | 1 + src/py/crankshaft/test/test_cluster_kmeans.py | 38 ++++++++++++ 8 files changed, 166 insertions(+) create mode 100644 doc/11_kmeans.md create mode 100644 src/pg/sql/11_kmeans.sql create mode 100644 src/pg/test/expected/05_kmeans_test.out create mode 100644 src/pg/test/sql/05_kmeans_test.sql create mode 100644 src/py/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 src/py/crankshaft/test/fixtures/kmeans.json create mode 100644 src/py/crankshaft/test/test_cluster_kmeans.py diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md new file mode 100644 index 0000000..6153010 --- /dev/null +++ b/doc/11_kmeans.md @@ -0,0 +1,62 @@ +## K-Means Functions + +### CDB_KMeans(subquery text, no_clusters INTEGER) + +This function attempts to find n clusters within the input data. It will return a table to CartoDB ids and +the number of the cluster each point in the input was assigend to. + + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| no\_clusters | INTEGER | The number of clusters to try and find | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| cartodb\_id | INTEGER | The CartoDB id of the row in the input table.| +| cluster\_no | INTEGER | The cluster that this point belongs to. | + + +#### Example Usage + +```sql +SELECT + customers.*, + km.cluster_no + FROM cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) km, customers_3 + WHERE customers.cartodb_id = km.cartodb_id +``` + +### CDB_WeightedMean(subquery text, weight_column text, category_column text) + +Function that computes the weighted centroid of a number of clusters by some weight column. + +### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column and the columns specified as the weight and category columns| +| weight\_column | TEXT | The name of the column to use as a weight | +| category\_column | TEXT | The name of the column to use as a category | + +### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| the\_geom | GEOMETRY | A point for the weighted cluster center | +| class | INTEGER | The cluster class | + +### Example Usage + +```sql +SELECT ST_TRANSFORM(the_geom, 3857) as the_geom_webmercator, class +FROM cdb_weighted_mean('SELECT *, customer_value FROM customers','customer_value','cluster_no') +``` diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql new file mode 100644 index 0000000..73e2f1d --- /dev/null +++ b/src/pg/sql/11_kmeans.sql @@ -0,0 +1,31 @@ +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + import plpy + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + +CREATE OR REPLACE FUNCTION CDB_WeightedMean(query text, weight_column text, category_column text default null ) +RETURNS table (the_geom geometry,class integer ) as $$ +BEGIN + +RETURN QUERY + EXECUTE format( $string$ + select ST_SETSRID(st_makepoint(cx, cy),4326) the_geom, class from ( + select + %I as class, + sum(st_x(the_geom)*%I)/sum(%I) cx, + sum(st_y(the_geom)*%I)/sum(%I) cy + from (%s) a + group by %I + ) q + + $string$, category_column, weight_column,weight_column,weight_column,weight_column,query, category_column + ) + using the_geom + RETURN; +END +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/05_kmeans_test.out b/src/pg/test/expected/05_kmeans_test.out new file mode 100644 index 0000000..4e6db09 --- /dev/null +++ b/src/pg/test/expected/05_kmeans_test.out @@ -0,0 +1,10 @@ +\pset format unaligned +\set ECHO all +SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); +clusters +2 +(1 row) +SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +clusters +52 +(1 row) diff --git a/src/pg/test/sql/05_kmeans_test.sql b/src/pg/test/sql/05_kmeans_test.sql new file mode 100644 index 0000000..a400e5e --- /dev/null +++ b/src/pg/test/sql/05_kmeans_test.sql @@ -0,0 +1,6 @@ +\pset format unaligned +\set ECHO all + +SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); + +SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); diff --git a/src/py/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py index 0df080f..338e8ea 100644 --- a/src/py/crankshaft/crankshaft/clustering/__init__.py +++ b/src/py/crankshaft/crankshaft/clustering/__init__.py @@ -1 +1,2 @@ from moran import * +from kmeans import * diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..3d9ed58 --- /dev/null +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,17 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/src/py/crankshaft/test/fixtures/kmeans.json b/src/py/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/src/py/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/src/py/crankshaft/test/test_cluster_kmeans.py b/src/py/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/src/py/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + From 69f08c4b78bedd746e918897c2c015aa634fe0d1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 8 Jun 2016 11:26:32 -0400 Subject: [PATCH 068/183] updated tests to check for percent diffs --- src/pg/test/expected/05_markov_test.out | 57 +++---------------------- src/pg/test/sql/05_markov_test.sql | 29 +++++++++---- 2 files changed, 25 insertions(+), 61 deletions(-) diff --git a/src/pg/test/expected/05_markov_test.out b/src/pg/test/expected/05_markov_test.out index fd69a67..2f8d510 100644 --- a/src/pg/test/expected/05_markov_test.out +++ b/src/pg/test/expected/05_markov_test.out @@ -3,55 +3,8 @@ SET client_min_messages TO WARNING; _cdb_random_seeds (1 row) -cartodb_id|trend|trend_up|trend_down|volatility -1|0.0714285714286|0.0666666666667|0.0|0.367574633389 -2|0.222222222222|0.181818181818|0.0|0.317010832258 -3|0.0526315789474|0.05|0.0|0.37549966711 -4|0.1|0.235294117647|0.176470588235|0.215650620939 -5|0.0|0.08|0.08|0.32199378876 -6|-0.056338028169|0.0|0.0533333333333|0.373904325255 -7|0.121212121212|0.108108108108|0.0|0.348470518082 -8|0.0|0.0857142857143|0.0857142857143|0.316614746907 -9|-0.0416666666667|0.0967741935484|0.129032258065|0.291681407883 -10|0.222222222222|0.181818181818|0.0|0.317010832258 -11|-0.294117647059|0.0|0.227272727273|0.299586491878 -12|-0.2|0.0|0.166666666667|0.323178657161 -13|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 -14|-0.12|0.0333333333333|0.133333333333|0.320416395752 -15|0.0526315789474|0.05|0.0|0.37549966711 -16|0.0|0.0857142857143|0.0857142857143|0.316614746907 -17|0.0|0.0|0.0|0.4 -18|0.0|0.0|0.0|0.4 -19|-0.056338028169|0.0|0.0533333333333|0.373904325255 -20|0.238095238095|0.214285714286|0.0357142857143|0.286249498707 -21|0.0|0.08|0.08|0.32199378876 -22|0.0714285714286|0.0666666666667|0.0|0.367574633389 -23|-0.04|0.138888888889|0.166666666667|0.256640210277 -24|-0.04|0.138888888889|0.166666666667|0.256640210277 -25|0.0|0.08|0.08|0.32199378876 -26|0.1|0.235294117647|0.176470588235|0.215650620939 -27|-0.056338028169|0.0|0.0533333333333|0.373904325255 -28|-0.056338028169|0.0|0.0533333333333|0.373904325255 -29|0.0526315789474|0.05|0.0|0.37549966711 -30|-0.056338028169|0.0|0.0533333333333|0.373904325255 -31|0.238095238095|0.214285714286|0.0357142857143|0.286249498707 -32|-0.235294117647|0.0434782608696|0.217391304348|0.281234115283 -33|-0.2|0.0|0.166666666667|0.323178657161 -34|-0.2|0.0|0.166666666667|0.323178657161 -35|-0.12|0.0333333333333|0.133333333333|0.320416395752 -36|0.121212121212|0.108108108108|0.0|0.348470518082 -37|-0.056338028169|0.0|0.0533333333333|0.373904325255 -38|0.0526315789474|0.05|0.0|0.37549966711 -39|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 -40|-0.0416666666667|0.0967741935484|0.129032258065|0.291681407883 -41|0.0|0.0857142857143|0.0857142857143|0.316614746907 -42|0.222222222222|0.181818181818|0.0|0.317010832258 -43|0.0|0.0|0.0|0.4 -44|0.0|0.0|0.0|0.4 -45|0.0|0.166666666667|0.166666666667|0.244948974278 -46|0.1|0.0909090909091|0.0|0.356289417132 -47|-0.0769230769231|0.131578947368|0.184210526316|0.252741204285 -48|-0.333333333333|0.0|0.25|0.291547594742 -(48 rows) -cartodb_id|trend|trend_up|trend_down|volatility -(0 rows) +cartodb_id|trend_test|trend_up_test|trend_down_test|volatility_test +1|t|t|t|t +2|t|t|t|t +3|t|t|t|t +(3 rows) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 24581f9..06ccaef 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -8,15 +8,26 @@ SET client_min_messages TO WARNING; -- that affect those results to have repeatable results SELECT cdb_crankshaft._cdb_random_seeds(1234); -SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility +SELECT + m1.cartodb_id, + CASE WHEN m1.cartodb_id = 1 THEN (m2.trend - 0.0714285714286) / 0.0714285714286 < 0.01 + WHEN m1.cartodb_id = 2 THEN (m2.trend - 0.222222222222) / 0.222222222222 < 0.01 + WHEN m1.cartodb_id = 3 THEN (m2.trend - 0.0526315789474) / 0.0526315789474 < 0.01 + ELSE NULL END As trend_test, + CASE WHEN m1.cartodb_id = 1 THEN (m2.trend_up - 0.0666666666667) / 0.0666666666667 < 0.01 + WHEN m1.cartodb_id = 2 THEN (m2.trend_up - 0.181818181818) / 0.181818181818 < 0.01 + WHEN m1.cartodb_id = 3 THEN (m2.trend_up - 0.05) / 0.05 < 0.01 + ELSE NULL END As trend_up_test, + CASE WHEN m1.cartodb_id = 1 THEN m2.trend_down = 0.0 + WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0 + WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0 + ELSE NULL END As trend_down_test, + CASE WHEN m1.cartodb_id = 1 THEN (m2.volatility - 0.367574633389) / 0.367574633389 < 0.01 + WHEN m1.cartodb_id = 2 THEN (m2.volatility - 0.317010832258) / 0.317010832258 < 0.01 + WHEN m1.cartodb_id = 3 THEN (m2.volatility - 0.37549966711) / 0.37549966711 < 0.01 + ELSE NULL END As volatility_test FROM markov_usjoin_example As m1 JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 ON m1.cartodb_id = m2.rowid - ORDER BY m1.cartodb_id; - - - SELECT m1.cartodb_id, m2.trend, m2.trend_up, m2.trend_down, m2.volatility - FROM markov_usjoin_example As m1 - JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example LIMIT 0', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 - ON m1.cartodb_id = m2.rowid - ORDER BY m1.cartodb_id; +ORDER BY m1.cartodb_id +LIMIT 3; From 7f3b23f67a958faa9162efef2362cc13a1e665ff Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:06:49 +0000 Subject: [PATCH 069/183] reworking CDB_WeightedMean to be an aggregate function --- src/pg/sql/11_kmeans.sql | 60 +++++++++++++++++-------- src/pg/test/expected/05_kmeans_test.out | 2 +- src/pg/test/sql/05_kmeans_test.sql | 2 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 73e2f1d..87f07ea 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -8,24 +8,46 @@ RETURNS table (cartodb_id integer, cluster_no integer) as $$ $$ language plpythonu; -CREATE OR REPLACE FUNCTION CDB_WeightedMean(query text, weight_column text, category_column text default null ) -RETURNS table (the_geom geometry,class integer ) as $$ -BEGIN -RETURN QUERY - EXECUTE format( $string$ - select ST_SETSRID(st_makepoint(cx, cy),4326) the_geom, class from ( - select - %I as class, - sum(st_x(the_geom)*%I)/sum(%I) cx, - sum(st_y(the_geom)*%I)/sum(%I) cy - from (%s) a - group by %I - ) q - - $string$, category_column, weight_column,weight_column,weight_column,weight_column,query, category_column - ) - using the_geom - RETURN; -END +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(the_geom geometry(Point, 4326), weight NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); + + diff --git a/src/pg/test/expected/05_kmeans_test.out b/src/pg/test/expected/05_kmeans_test.out index 4e6db09..8c6ffa1 100644 --- a/src/pg/test/expected/05_kmeans_test.out +++ b/src/pg/test/expected/05_kmeans_test.out @@ -4,7 +4,7 @@ SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('se clusters 2 (1 row) -SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; clusters 52 (1 row) diff --git a/src/pg/test/sql/05_kmeans_test.sql b/src/pg/test/sql/05_kmeans_test.sql index a400e5e..2298b85 100644 --- a/src/pg/test/sql/05_kmeans_test.sql +++ b/src/pg/test/sql/05_kmeans_test.sql @@ -3,4 +3,4 @@ SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); -SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; From 9d3de5a8ef13be63539248f3e4d82d7b4b68df9d Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:12:55 +0000 Subject: [PATCH 070/183] adding not null filter for geom on kmeans --- src/py/crankshaft/crankshaft/clustering/kmeans.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py index 3d9ed58..4134062 100644 --- a/src/py/crankshaft/crankshaft/clustering/kmeans.py +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -5,6 +5,7 @@ def kmeans(query, no_clusters, no_init=20): data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, array_agg(ST_X(the_geom) order by cartodb_id) xs, array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null '''.format(query=query)) xs = data[0]['xs'] From 1a4944b9600250a972458bfe1952f79ffce76ff2 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:16:16 +0000 Subject: [PATCH 071/183] adding sklearn as a dep --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 8d5e622..baa88e3 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1'], + install_requires=['pysal==1.9.1', 'sklearn==0.17.1'], - requires=['pysal', 'numpy' ], + requires=['pysal', 'numpy', 'sklearn' ], test_suite='test' ) From 889cd5c5791d2f87e35b3e510b7c7ac14eac9fcf Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jun 2016 17:47:46 +0200 Subject: [PATCH 072/183] Fix scikit-learn dep name --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index baa88e3..68f9e17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'sklearn==0.17.1'], + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn' ], From b33ba2d2949ab0bef092f25acf82d2308775a2a5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jun 2016 18:24:43 +0200 Subject: [PATCH 073/183] Do not use names for the aggregate params --- src/pg/sql/11_kmeans.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 87f07ea..a27f803 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -43,11 +43,9 @@ BEGIN END $$ LANGUAGE plpgsql; -CREATE AGGREGATE CDB_WeightedMean(the_geom geometry(Point, 4326), weight NUMERIC)( +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( SFUNC = CDB_WeightedMeanS, FINALFUNC = CDB_WeightedMeanF, STYPE = Numeric[], INITCOND = "{0.0,0.0,0.0}" ); - - From 1e8bc12e0a6ea2ffefe580b63133b88f4db045a7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 12:17:46 +0200 Subject: [PATCH 074/183] Declare scipy as dep --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 68f9e17..e787d32 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['pysal', 'numpy', 'sklearn' ], + requires=['scipy', 'pysal', 'numpy', 'sklearn'], test_suite='test' ) From c870f68c77652a11f8401bbbb981797694174288 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 13:05:50 +0200 Subject: [PATCH 075/183] Revert "Declare scipy as dep" This reverts commit 1e8bc12e0a6ea2ffefe580b63133b88f4db045a7. --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index e787d32..68f9e17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['scipy', 'pysal', 'numpy', 'sklearn'], + requires=['pysal', 'numpy', 'sklearn' ], test_suite='test' ) From fd1862167c123ad7e59906801027e06c88fbf90e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 13:06:21 +0200 Subject: [PATCH 076/183] Remove trailing space --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 68f9e17..04822dd 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -42,7 +42,7 @@ setup( # provisioned in the production servers. install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['pysal', 'numpy', 'sklearn' ], + requires=['pysal', 'numpy', 'sklearn'], test_suite='test' ) From 7b98415da318e5dd5119e7c10b5b0b2ca54f3c8d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:06:23 +0200 Subject: [PATCH 077/183] Remove virtualenv activation #60 --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- src/pg/sql/02_py.sql | 23 ----------------------- src/pg/sql/03_random_seeds.sql | 1 - src/pg/sql/10_moran.sql | 4 ---- 4 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 src/pg/sql/02_py.sql diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 882cece..9bb2e75 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ - [ ] All declared geometries are `geometry(Geometry, 4326)` for general geoms, or `geometry(Point, 4326)` -- [ ] Include python is activated for new functions. Include this before importing modules: `plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')` +- [ ] Existing functions in crankshaft python library called from the extension are kept at least from version N to version N+1 (to avoid breakage during upgrades). - [ ] Docs for public-facing functions are written - [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`. - [ ] If appropriate, new functions accepts an arbitrary query as an input (see [Crankshaft Issue #6](https://github.com/CartoDB/crankshaft/issues/6) for more information) diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql deleted file mode 100644 index 7da5f47..0000000 --- a/src/pg/sql/02_py.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() -RETURNS text -AS $$ - BEGIN - -- RETURN '/opt/virtualenvs/crankshaft'; - RETURN '@@VIRTUALENV_PATH@@'; - END; -$$ language plpgsql IMMUTABLE STRICT; - --- Use the crankshaft python module -CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() -RETURNS VOID -AS $$ - import os - # plpy.notice('%',str(os.environ)) - # activate virtualenv - crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] - base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] - default_venv_path = os.path.join(base_path, crankshaft_version) - venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) - activate_path = venv_path + '/bin/activate_this.py' - exec(open(activate_path).read(), dict(__file__=activate_path)) -$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/03_random_seeds.sql b/src/pg/sql/03_random_seeds.sql index 9a0cca6..2b62be3 100644 --- a/src/pg/sql/03_random_seeds.sql +++ b/src/pg/sql/03_random_seeds.sql @@ -4,7 +4,6 @@ CREATE OR REPLACE FUNCTION _cdb_random_seeds (seed_value INTEGER) RETURNS VOID AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft import random_seeds random_seeds.set_random_seeds(seed_value) $$ LANGUAGE plpythonu; diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index a336867..3be31a2 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -10,7 +10,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, significance NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -28,7 +27,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT) RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -122,7 +120,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran FLOAT, significance FLOAT) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -143,7 +140,6 @@ CREATE OR REPLACE FUNCTION RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate # TODO: use named parameters or a dictionary return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) From 0acae8240f777e042f59dfcf3f0a3e1430dcb984 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:23:30 +0200 Subject: [PATCH 078/183] Remove virtualenv stuff from Makefiles #60 --- Makefile | 6 ------ src/pg/Makefile | 7 +------ src/py/Makefile | 11 +++-------- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 6c3e219..ef9415b 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ PYP_DIR = src/py # Generate and install developmet versions of the extension # and python package. # The extension is named 'dev' with a 'current' alias for easily upgrading. -# The Python package is installed in a virtual environment envs/dev/ # Requires sudo. install: ## Generate and install development version of the extension; requires sudo. $(MAKE) -C $(PYP_DIR) install @@ -29,7 +28,6 @@ release: ## Generate a new release of the extension. Only for telease manager $(MAKE) -C $(PYP_DIR) release # Install the current release. -# The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. # Use the RELEASE_VERSION environment variable to deploy a specific version: # sudo make deploy RELEASE_VERSION=1.0.0 @@ -52,10 +50,6 @@ clean-release: ## clean up current release rm -rf release/python/$(RELEASE_VERSION) rm -f release/$(RELEASE_VERSION)--*.sql -# Cleanup all virtual environments -clean-environments: ## clean up all virtual environments - rm -rf envs/* - clean-all: clean-dev clean-release clean-environments help: diff --git a/src/pg/Makefile b/src/pg/Makefile index 8a745c4..178ed08 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -7,7 +7,6 @@ include ../../Makefile.global # requires sudo. In additionof the current development version # named 'dev', an alias 'current' is generating for ease of # update (upgrade to 'current', then to 'dev'). -# the python module is installed in a virtualenv in envs/dev/ # * test runs the tests for the currently generated Development # extension. @@ -18,11 +17,8 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../../envs) -ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) -REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ - -e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g' +REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' $(DATA): $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ @@ -54,7 +50,6 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql # Install the current relese into the PostgreSQL extensions directory -# and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' diff --git a/src/py/Makefile b/src/py/Makefile index 90b22b8..403c5a1 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -2,14 +2,11 @@ include ../../Makefile.global # Install the package locally for development install: - virtualenv --system-site-packages ../../envs/dev - # source ../../envs/dev/bin/activate - ../../envs/dev/bin/pip install -I ./crankshaft - ../../envs/dev/bin/pip install -I nose + pip install ./crankshaft # Test develpment install test: - ../../envs/dev/bin/nosetests crankshaft/test/ + nosetests crankshaft/test/ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) mkdir -p ../../release/python/$(EXTVERSION) @@ -17,6 +14,4 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py deploy: - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose + pip install --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) From 75531b671e247b507d0a11d6f2fdced5ef3a8084 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:24:43 +0200 Subject: [PATCH 079/183] Remove virtualenv references from READMEs #60 --- README.md | 3 +-- src/py/README.md | 17 +---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 68a64fb..0ff9090 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,10 @@ CartoDB Spatial Analysis extension for PostgreSQL. * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code * *release* reseleased versions -* *env* base directory for Python virtual environments ## Requirements -* pip, virtualenv, PostgreSQL +* pip, PostgreSQL * python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/master/src/py/README.md)) # Working Process -- Quickstart Guide diff --git a/src/py/README.md b/src/py/README.md index 29a3145..8fcfcb7 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -10,7 +10,6 @@ nosetests test/ ## Notes about Python dependencies * This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments. -* We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has all the dependencies * Every dependency should be: - Added to the `setup.py` file - Installed through it @@ -30,21 +29,7 @@ PySAL 1.10 or later, so we'll stick to 1.9.1. apt-get install -y python-scipy ``` -We'll use virtual environments to install our packages, -but configued to use also system modules so that the -mentioned scipy and numpy are used. - - # Create a virtual environment for python - $ virtualenv --system-site-packages dev - - # Activate the virtualenv - $ source dev/bin/activate - - # Install all the requirements - # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -I ./crankshaft - -#### Test the libraries with that virtual env +#### Test the libraries ##### Test numpy library dependency: From a8943bae985acc4d960d7cb614c5e6ad4bb68ed1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:27:35 +0200 Subject: [PATCH 080/183] Remove reference to clean-environments #60 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef9415b..50f690c 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ clean-release: ## clean up current release rm -rf release/python/$(RELEASE_VERSION) rm -f release/$(RELEASE_VERSION)--*.sql -clean-all: clean-dev clean-release clean-environments +clean-all: clean-dev clean-release help: @IFS=$$'\n' ; \ From bbe22d0b4dd0c4deadc0f8741a54f315ae7cc8e0 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 15 Jun 2016 11:36:04 -0400 Subject: [PATCH 081/183] new bounds for accuracy in tests --- src/pg/test/sql/05_markov_test.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 06ccaef..ad2e053 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -22,9 +22,9 @@ SELECT WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0 WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0 ELSE NULL END As trend_down_test, - CASE WHEN m1.cartodb_id = 1 THEN (m2.volatility - 0.367574633389) / 0.367574633389 < 0.01 - WHEN m1.cartodb_id = 2 THEN (m2.volatility - 0.317010832258) / 0.317010832258 < 0.01 - WHEN m1.cartodb_id = 3 THEN (m2.volatility - 0.37549966711) / 0.37549966711 < 0.01 + CASE WHEN m1.cartodb_id = 1 THEN (m2.volatility - 0.367574633389) / 0.367574633389 < 0.1 + WHEN m1.cartodb_id = 2 THEN (m2.volatility - 0.317010832258) / 0.317010832258 < 0.1 + WHEN m1.cartodb_id = 3 THEN (m2.volatility - 0.37549966711) / 0.37549966711 < 0.1 ELSE NULL END As volatility_test FROM markov_usjoin_example As m1 JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 From 4834ee2f4212b77457636e7582da2a3c9370d0e7 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 15 Jun 2016 11:36:19 -0400 Subject: [PATCH 082/183] adding higher numpy version --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 8d5e622..e58add1 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1'], + install_requires=['pysal==1.9.1', 'numpy==1.11.0'], requires=['pysal', 'numpy' ], From d08a2b6d2d756be58a16e80bf4ded3d134dfb97a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 14:12:28 +0200 Subject: [PATCH 083/183] Remove _cdb_crankshaft_activate_py activation call from kmeans function --- src/pg/sql/11_kmeans.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index a27f803..125aac3 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -1,8 +1,6 @@ CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) RETURNS table (cartodb_id integer, cluster_no integer) as $$ - import plpy - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import kmeans return kmeans(query,no_clusters,no_init) From 8b5e9102345fc2a7218961ef26c033715a441d6b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 14:16:32 +0200 Subject: [PATCH 084/183] Release 0.0.3 --- NEWS.md | 5 + release/crankshaft--0.0.2--0.0.3.sql | 413 ++++++++++++++++++ release/crankshaft--0.0.3--0.0.2.sql | 209 +++++++++ release/crankshaft--0.0.3.sql | 403 +++++++++++++++++ release/crankshaft.control | 2 +- .../0.0.3/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 2 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 260 +++++++++++ .../crankshaft/pysal_utils/__init__.py | 1 + .../crankshaft/pysal_utils/pysal_utils.py | 152 +++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.3/crankshaft/setup.py | 48 ++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../0.0.3/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.3/crankshaft/test/helper.py | 13 + .../python/0.0.3/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 ++ .../crankshaft/test/test_clustering_moran.py | 83 ++++ .../0.0.3/crankshaft/test/test_pysal_utils.py | 107 +++++ src/pg/crankshaft.control | 2 +- 22 files changed, 1907 insertions(+), 2 deletions(-) create mode 100644 release/crankshaft--0.0.2--0.0.3.sql create mode 100644 release/crankshaft--0.0.3--0.0.2.sql create mode 100644 release/crankshaft--0.0.3.sql create mode 100644 release/python/0.0.3/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.3/crankshaft/setup.py create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.3/crankshaft/test/helper.py create mode 100644 release/python/0.0.3/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.0.3/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.0.3/crankshaft/test/test_pysal_utils.py diff --git a/NEWS.md b/NEWS.md index 0b8c2da..ed66fd9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.0.3 (2016-06-16) +------------------ +* Adds new functions: kmeans, weighted centroids. +* Replaces moran functions with new areas of interest naming. + 0.0.2 (2016-03-16) ------------------ * New versioning approach using per-version Python virtual environments diff --git a/release/crankshaft--0.0.2--0.0.3.sql b/release/crankshaft--0.0.2--0.0.3.sql new file mode 100644 index 0000000..8a865d5 --- /dev/null +++ b/release/crankshaft--0.0.2--0.0.3.sql @@ -0,0 +1,413 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-- [MANUALLY] DROP FUNCTIONS REMOVED SINCE 0.0.2 version + +DROP FUNCTION IF EXISTS cdb_moran_local(TEXT, TEXT, float, INT, INT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_moran_local_rate(TEXT, TEXT, TEXT, FLOAT, INT, INT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS _cdb_crankshaft_virtualenvs_path(); +DROP FUNCTION IF EXISTS _cdb_crankshaft_activate_py(); + +-- [END MANUALLY] DROP FUNCTIONS REMOVED SINCE 0.0.2 version + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() + RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() + RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION + _cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) + RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) + RETURNS Numeric[] AS + $$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) + RETURNS GEOMETRY AS + $$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( +SFUNC = CDB_WeightedMeanS, +FINALFUNC = CDB_WeightedMeanF, +STYPE = Numeric[], +INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS + $$ + DECLARE + result numeric; + qualified_name text; + BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; + END; + $$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) + RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft--0.0.3--0.0.2.sql b/release/crankshaft--0.0.3--0.0.2.sql new file mode 100644 index 0000000..a2ccd2f --- /dev/null +++ b/release/crankshaft--0.0.3--0.0.2.sql @@ -0,0 +1,209 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-- [MANUALLY] DROP FUNCTIONS INTRODUCED IN 0.0.3 version + +DROP FUNCTION IF EXISTS CDB_AreasOfInterestGlobal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS _CDB_AreasOfInterestLocal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestLocal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialHotspots(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialColdspots(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialOutliers(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestGlobalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestLocalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS _CDB_AreasOfInterestLocalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialHotspotsRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialColdspotsRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialOutliersRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_KMeans(text,integer,integer); +DROP AGGREGATE IF EXISTS CDB_WeightedMean(geometry(Point, 4326), NUMERIC); +DROP FUNCTION IF EXISTS CDB_WeightedMeanS(Numeric[], GEOMETRY(Point, 4326), NUMERIC); +DROP FUNCTION IF EXISTS CDB_WeightedMeanF(Numeric[]); + + +-- [END MANUALLY] DROP FUNCTIONS INTRODUCED IN 0.0.3 version + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.2'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + -- RETURN '/opt/virtualenvs/crankshaft'; + RETURN '/home/ubuntu/crankshaft/envs'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I +CREATE OR REPLACE FUNCTION + cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate +CREATE OR REPLACE FUNCTION + cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft--0.0.3.sql b/release/crankshaft--0.0.3.sql new file mode 100644 index 0000000..caacd75 --- /dev/null +++ b/release/crankshaft--0.0.3.sql @@ -0,0 +1,403 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 49c0d22..2029b7e 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.2' +default_version = '0.0.3' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.0.3/crankshaft/crankshaft/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..338e8ea --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,2 @@ +from moran import * +from kmeans import * diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..39b3ff6 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,260 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": attr_name, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": numerator, + "attr2": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + query = pu.construct_neighbor_query(w_type, + {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs}) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find units of significance + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "subquery": subquery, + "geom_col": geom_col, + "id_col": id_col} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..835880d --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1 @@ +from pysal_utils import * diff --git a/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..02b5e35 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,152 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res: query results with attributes and neighbors + """ + if w_type.lower() == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + else: + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.3/crankshaft/setup.py b/release/python/0.0.3/crankshaft/setup.py new file mode 100644 index 0000000..33a3b62 --- /dev/null +++ b/release/python/0.0.3/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.3', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json b/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.0.3/crankshaft/test/fixtures/moran.json b/release/python/0.0.3/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.3/crankshaft/test/helper.py b/release/python/0.0.3/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.3/crankshaft/test/mock_plpy.py b/release/python/0.0.3/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py b/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.0.3/crankshaft/test/test_clustering_moran.py b/release/python/0.0.3/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..393e93b --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,83 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.0.3/crankshaft/test/test_pysal_utils.py b/release/python/0.0.3/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..4ea0d9b --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,107 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(pu.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 49c0d22..2029b7e 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.2' +default_version = '0.0.3' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft From 1e19f468ebfc0626c37b85fd88ad0d1da1531771 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 16:23:43 +0200 Subject: [PATCH 085/183] Declare numpy dep --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 04822dd..f072f17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['numpy==1.11.0', 'pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn'], From 237aa1c5818f003ffb459817ea5e72392c765c5c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 16:34:45 +0200 Subject: [PATCH 086/183] Declare scipy as dep --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index f072f17..266b6f1 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['numpy==1.11.0', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['numpy==1.11.0', 'scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn'], From 3480a0d252b1b7f9e79397b126b08f65837d3036 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Thu, 16 Jun 2016 16:56:16 +0200 Subject: [PATCH 087/183] Allow passing options to pip install --- src/py/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/Makefile b/src/py/Makefile index 403c5a1..b584645 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -13,5 +13,5 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) cp -r ./$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py -deploy: - pip install --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) +deploy: + pip install $(RUN_OPTIONS) --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) From 1db938c450634532133190b3c28425e7313acc72 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 19:07:42 +0200 Subject: [PATCH 088/183] Removes cartodb-extension-dep --- CONTRIBUTING.md | 1 - src/pg/crankshaft.control | 2 +- src/pg/test/expected/01_install_test.out | 1 - src/pg/test/sql/01_install_test.sql | 1 - src/pg/test/sql/90_permissions.sql | 2 +- 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f642d45..42385dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,6 @@ it can be installed directly with: * `CREATE EXTENSION IF NOT EXISTS plpythonu;` `CREATE EXTENSION IF NOT EXISTS postgis;` - `CREATE EXTENSION IF NOT EXISTS cartodb;` `CREATE EXTENSION crankshaft WITH VERSION 'dev';` Note: the development extension uses the development python virtual diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 2029b7e..e71321f 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' default_version = '0.0.3' -requires = 'plpythonu, postgis, cartodb' +requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index e40d267..e84a48a 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,6 +1,5 @@ -- Install dependencies CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index fc3ea80..bbce805 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,7 +1,6 @@ -- Install dependencies CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/src/pg/test/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql index 187f795..1e9ea99 100644 --- a/src/pg/test/sql/90_permissions.sql +++ b/src/pg/test/sql/90_permissions.sql @@ -4,7 +4,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SET ROLE test_regular_user; -- Add to the search path the schema -SET search_path TO public,cartodb,cdb_crankshaft; +SET search_path TO public,cdb_crankshaft; -- Exercise public functions SELECT ppoints.code, m.quads From f5fb4499db226521adb952b7524449ae15ddcc3a Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 20 Jun 2016 09:44:52 +0200 Subject: [PATCH 089/183] Set final dependencies versions --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 266b6f1..abd4dae 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['numpy==1.11.0', 'scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], requires=['pysal', 'numpy', 'sklearn'], From 01fc2c1dd1087e58679cc2728ee88a0df5702ea6 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 20 Jun 2016 10:04:22 +0200 Subject: [PATCH 090/183] Release 0.0.4 --- NEWS.md | 5 + release/crankshaft--0.0.3--0.0.4.sql | 8 + release/crankshaft--0.0.4--0.0.3.sql | 8 + release/crankshaft--0.0.4.sql | 403 ++++++++++++++++++ release/crankshaft.control | 4 +- .../0.0.4/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 2 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 260 +++++++++++ .../crankshaft/pysal_utils/__init__.py | 1 + .../crankshaft/pysal_utils/pysal_utils.py | 152 +++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.4/crankshaft/setup.py | 48 +++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../0.0.4/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.4/crankshaft/test/helper.py | 13 + .../python/0.0.4/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 ++ .../crankshaft/test/test_clustering_moran.py | 83 ++++ .../0.0.4/crankshaft/test/test_pysal_utils.py | 107 +++++ src/pg/crankshaft.control | 2 +- 22 files changed, 1302 insertions(+), 3 deletions(-) create mode 100644 release/crankshaft--0.0.3--0.0.4.sql create mode 100644 release/crankshaft--0.0.4--0.0.3.sql create mode 100644 release/crankshaft--0.0.4.sql create mode 100644 release/python/0.0.4/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.4/crankshaft/setup.py create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.4/crankshaft/test/helper.py create mode 100644 release/python/0.0.4/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.0.4/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.0.4/crankshaft/test/test_pysal_utils.py diff --git a/NEWS.md b/NEWS.md index ed66fd9..c011a0d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.0.4 (2016-06-20) +------------------ +* Remove cartodb extension dependency from tests +* Declare all correct dependencies with correct versions in setup.py + 0.0.3 (2016-06-16) ------------------ * Adds new functions: kmeans, weighted centroids. diff --git a/release/crankshaft--0.0.3--0.0.4.sql b/release/crankshaft--0.0.3--0.0.4.sql new file mode 100644 index 0000000..69038a3 --- /dev/null +++ b/release/crankshaft--0.0.3--0.0.4.sql @@ -0,0 +1,8 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.4'::text; +$$ language 'sql' STABLE STRICT; diff --git a/release/crankshaft--0.0.4--0.0.3.sql b/release/crankshaft--0.0.4--0.0.3.sql new file mode 100644 index 0000000..bd8ed82 --- /dev/null +++ b/release/crankshaft--0.0.4--0.0.3.sql @@ -0,0 +1,8 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; diff --git a/release/crankshaft--0.0.4.sql b/release/crankshaft--0.0.4.sql new file mode 100644 index 0000000..c855958 --- /dev/null +++ b/release/crankshaft--0.0.4.sql @@ -0,0 +1,403 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.4'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 2029b7e..01088b1 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.3' -requires = 'plpythonu, postgis, cartodb' +default_version = '0.0.4' +requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.0.4/crankshaft/crankshaft/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..338e8ea --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,2 @@ +from moran import * +from kmeans import * diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..39b3ff6 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,260 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": attr_name, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": numerator, + "attr2": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + query = pu.construct_neighbor_query(w_type, + {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs}) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find units of significance + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "subquery": subquery, + "geom_col": geom_col, + "id_col": id_col} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..835880d --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1 @@ +from pysal_utils import * diff --git a/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..02b5e35 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,152 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res: query results with attributes and neighbors + """ + if w_type.lower() == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + else: + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.4/crankshaft/setup.py b/release/python/0.0.4/crankshaft/setup.py new file mode 100644 index 0000000..32d1ead --- /dev/null +++ b/release/python/0.0.4/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.4', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json b/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.0.4/crankshaft/test/fixtures/moran.json b/release/python/0.0.4/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.4/crankshaft/test/helper.py b/release/python/0.0.4/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.4/crankshaft/test/mock_plpy.py b/release/python/0.0.4/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py b/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.0.4/crankshaft/test/test_clustering_moran.py b/release/python/0.0.4/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..393e93b --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,83 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.0.4/crankshaft/test/test_pysal_utils.py b/release/python/0.0.4/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..4ea0d9b --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,107 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(pu.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index e71321f..01088b1 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.3' +default_version = '0.0.4' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft From 79699cd5cb4c6d1283ccdd2dbbdacdbbec6e560f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 20 Jun 2016 17:27:37 -0400 Subject: [PATCH 091/183] changing to new crankshaft format --- {pg/sql/0.0.1 => src/pg/sql}/05_segmentation.sql | 8 ++++---- src/py/crankshaft/crankshaft/__init__.py | 1 + .../py}/crankshaft/crankshaft/segmentation/__init__.py | 0 .../crankshaft/crankshaft/segmentation/segmentation.py | 0 4 files changed, 5 insertions(+), 4 deletions(-) rename {pg/sql/0.0.1 => src/pg/sql}/05_segmentation.sql (93%) rename {python => src/py}/crankshaft/crankshaft/segmentation/__init__.py (100%) rename {python => src/py}/crankshaft/crankshaft/segmentation/segmentation.py (100%) diff --git a/pg/sql/0.0.1/05_segmentation.sql b/src/pg/sql/05_segmentation.sql similarity index 93% rename from pg/sql/0.0.1/05_segmentation.sql rename to src/pg/sql/05_segmentation.sql index b9bca6a..00ced3f 100644 --- a/pg/sql/0.0.1/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -1,5 +1,5 @@ CREATE OR REPLACE FUNCTION - cdb_create_segment ( + CDB_CreateSegment ( segment_name TEXT, table_name TEXT, column_name TEXT, @@ -14,7 +14,7 @@ AS $$ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION - cdb_correlated_variables( + CDB_CorrelatedVariables( query text, geoid_column text DEFAULT 'geoid', census_table text DEFAULT 'ml_learning_block_groups_clipped' @@ -26,7 +26,7 @@ AS $$ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION - cdb_predict_segment ( + CDB_PredictSegment ( segment_name TEXT, geoid_column TEXT DEFAULT 'geoid', census_table TEXT DEFAULT 'block_groups' @@ -40,7 +40,7 @@ $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION - cdb_create_and_predict_segment ( + CDB_CreateAndPredictSegment ( segment_name TEXT, query TEXT, target_table TEXT, diff --git a/src/py/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py index d07e330..bc8e065 100644 --- a/src/py/crankshaft/crankshaft/__init__.py +++ b/src/py/crankshaft/crankshaft/__init__.py @@ -1,2 +1,3 @@ import random_seeds import clustering +import segmentation diff --git a/python/crankshaft/crankshaft/segmentation/__init__.py b/src/py/crankshaft/crankshaft/segmentation/__init__.py similarity index 100% rename from python/crankshaft/crankshaft/segmentation/__init__.py rename to src/py/crankshaft/crankshaft/segmentation/__init__.py diff --git a/python/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py similarity index 100% rename from python/crankshaft/crankshaft/segmentation/segmentation.py rename to src/py/crankshaft/crankshaft/segmentation/segmentation.py From 1d13b98d68d2e6e621d7225080a9cbdbdd79ecf8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 21 Jun 2016 21:00:30 +0000 Subject: [PATCH 092/183] Basic version of create and predict segment --- src/pg/sql/05_segmentation.sql | 53 +---- .../crankshaft/segmentation/segmentation.py | 197 +++++------------- 2 files changed, 52 insertions(+), 198 deletions(-) diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 00ced3f..7def90d 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -1,55 +1,12 @@ -CREATE OR REPLACE FUNCTION - CDB_CreateSegment ( - segment_name TEXT, - table_name TEXT, - column_name TEXT, - geoid_column TEXT DEFAULT 'geoid', - census_table TEXT DEFAULT 'block_groups' - ) -RETURNS NUMERIC -AS $$ - from crankshaft import segmentation - # TODO: use named parameters or a dictionary - return segmentation.create_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION - CDB_CorrelatedVariables( - query text, - geoid_column text DEFAULT 'geoid', - census_table text DEFAULT 'ml_learning_block_groups_clipped' - ) -RETURNS TABLE(feature text, importance NUMERIC, std NUMERIC) -AS $$ - from crankshaft.segmentation import correlated_variables - return correlated_variables(query,geoid_column,census_table) -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION - CDB_PredictSegment ( - segment_name TEXT, - geoid_column TEXT DEFAULT 'geoid', - census_table TEXT DEFAULT 'block_groups' - ) -RETURNS TABLE(geoid TEXT, prediction NUMERIC) -AS $$ - from crankshaft.segmentation import create_segemnt - # TODO: use named parameters or a dictionary - return create_segment('table') -$$ LANGUAGE plpythonu; - - CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( - segment_name TEXT, query TEXT, - target_table TEXT, - geoid_column TEXT DEFAULT 'geoid', - census_table TEXT DEFAULT 'block_groups' + variable_name TEXT, + target_table TEXT ) -RETURNS TABLE (the_geom geometry, geoid text, prediction Numeric ) +RETURNS TABLE (cartodb_id text, prediction Numeric ) AS $$ - from crankshaft import segmentation + from crankshaft.segmentation import create_and_predict_segment # TODO: use named parameters or a dictionary - return segmentation.create_and_predict_segment(segment_name,query,geoid_column,census_table,target_table,'random_forest') + return create_and_predict_segment(query,variable_name,target_table) $$ LANGUAGE plpythonu; diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 0a1a8da..3881733 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -4,193 +4,90 @@ Segmentation creation and prediction import sklearn import numpy as np -import pandas as pd -import cPickle import plpy -import sys -from sklearn.ensemble import ExtraTreesRegressor +from sklearn.ensemble import GradientBoostingClassifier from sklearn import metrics -from sklearn.externals import joblib from sklearn.cross_validation import train_test_split -import StringIO -import gzip # High level interface --------------------------------------- -def create_segment(segment_name,table_name,column_name,geoid_column,census_table,method): +def replace_nan_with_mean(array): + indices = np.where(np.isnan(array)) #returns an array of rows and column indices + for row, col in zip(*indices): + array[row,col] = np.mean(array[~np.isnan(array[:,col]), col]) + return array + +def get_data(variable, feature_columns, query): + columns = ','.join(['array_agg("{col}") as "{col}"'.format(col=col) for col in feature_columns]) + data = plpy.execute(''' select array_agg("{variable}") as target, {columns} from ({query}) as a'''.format( + variable = variable, + columns = columns, + query = query + )) + target = np.array(data[0]['target']) + features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) + return replace_nan_with_mean(target), replace_nan_with_mean(features) + +def create_and_predict_segment(query,variable,target_query): """ generate a segment with machine learning Stuart Lynn """ - data = pd.DataFrame(join_with_census(table_name, column_name,geoid_column, census_table)) - features = data[data.columns.difference([column_name, 'geoid','the_geom', 'the_geom_webmercator'])] - target, mean, std = normalize(data[column_name]) + columns = plpy.execute('select * from ({query}) a limit 1 '.format(query=query))[0].keys() + + feature_columns = set(columns) - set([variable, 'the_geom', 'the_geom_webmercator']) + target,features = get_data(variable, feature_columns, query) + model, accuracy = train_model(target,features, test_split=0.2) - save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) - # predict_segment - return accuracy + cartodb_ids, result = predict_segment(model,feature_columns,target_query) + return zip(cartodb_ids, result) -def correlated_variables(query,geoid_column,census_table): - """ - returns the columns which are importaint for the random forrest model - """ - data = pd.DataFrame(join_with_census(query,geoid_column, census_table)) - features = data[data.columns.difference(['target', 'the_geom_webmercator', 'geoid','the_geom'])] - target, mean, std = normalize(data['target']) - model, accuracy, used_features = train_model(target,features, test_split=0.2) - std = np.std([tree.feature_importances_ for tree in model.estimators_], - axis=0) - importances = model.feature_importances_ - return zip(features,importances,std) - - -def create_and_predict_segment(segment_name,query,geoid_column,census_table,target_table,method): - """ - generate a segment with machine learning - Stuart Lynn - """ - data = pd.DataFrame(join_with_census(query,geoid_column, census_table)) - features = data[data.columns.difference(['target', 'the_geom_webmercator', 'geoid','the_geom'])] - target, mean, std = normalize(data['target']) - - normed_target,target_mean, target_std = normalize(target) - plpy.notice('mean ', target_mean, " std ", target_std) - model, accuracy, used_features = train_model(target,features, test_split=0.2) - # save_model(segment_name, model, accuracy, table_name, column_name, census_table, geoid_column, method) - geoms, geoids, result = predict_segment(model,used_features,geoid_column,target_table) - return zip(geoms,geoids, [denormalize(t,target_mean, target_std) for t in result] ) - - -def normalize(target): - mean = np.mean(target) - std = np.std(target) - plpy.notice('mean '+str(mean)+" std : "+str(std)) - return (target - mean)/std, mean, std - -def denormalize(target, mean ,std): - return target*std + mean def train_model(target,features,test_split): - plpy.notice('training the model') - plpy.notice('before ', str(np.shape(features))) - features = features.dropna(axis =1, how='all').fillna(0) - plpy.notice('after ', str(np.shape(features))) - target = target.fillna(0) features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) - model = ExtraTreesRegressor(n_estimators = 200, max_features=len(features.columns)) + model = GradientBoostingClassifier(n_estimators = 200, max_features=features.shape[1]) plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) - plpy.notice('training the model: fitting one') + plpy.notice('model trained') accuracy = calculate_model_accuracy(model,features,target) - return model, accuracy, features.columns + return model, accuracy def calculate_model_accuracy(model,features,target): prediction = model.predict(features) return metrics.mean_squared_error(prediction,target)/np.std(target) -def join_with_census(query, geoid_column, census_table): - columns = plpy.execute('select * from {census_table} limit 1 '.format(**locals())) - combined_columns = [ a for a in columns[0].keys() if a not in ['target','the_geom','cartodb_id','geoid','the_geom_webmercator']] - plpy.notice(combined_columns) - feature_names = ",".join([ " {census_table}.\"{a}\"::Numeric as \"{a}\" ".format(**locals()) for a in combined_columns]) - plpy.notice(feature_names) - - plpy.notice('joining with census data') - join_data = plpy.execute(''' - - SELECT {feature_names}, a.target - FROM ({query}) a - JOIN {census_table} - ON a.{geoid_column}::numeric = {census_table}.geoid::numeric - '''.format(**locals())) - - if len(join_data) == 0: - plpy.notice('Failed to join with census data') - - return query_to_dictionary(join_data) - -def query_to_dictionary(result): - return [ dict(zip(r.keys(), r.values())) for r in result ] - -def predict_segment(model,features,geoid_column,census_table): +def predict_segment(model,features,target_query): """ predict a segment with machine learning Stuart Lynn """ - # data = fetch_model(segment_name) - # model = data['model'] - # features = ",".join(features) - joined_features = ','.join(['\"'+a+'\"::numeric' for a in features]) - = plpy.execute() - cursor = plpy.cursor('select {joined_features} from {census_table}'.format(**locals())) + batch_size = 1000 + joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) + + cursor = plpy.cursor('select Array[{joined_features}] features from ({target_query}) a'.format( + joined_features=joined_features, + target_query= target_query + )) + results = [] + while True: rows = cursor.fetch(batch_size) if not rows: break + batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) - batch = pd.DataFrame(query_to_dictionary(rows)) - batch_features = batch.dropna(axis =1, how='all').fillna(0) - prediction = model.predict(batch_features) + #Need to fix this. Should be global mean. This will cause weird effects + batch = replace_nan_with_mean(batch) + plpy.notice(len(batch)) + prediction = model.predict(batch) results.append(prediction) plpy.notice('predicting: predicted') - return [a['the_geom'] for a in geoms], [a['geoid'] for a in geo_ids],prediction + + cartodb_ids = plpy.execute('select array_agg(cartodb_id order by cartodb_id) as cartodb_ids from ({0}) a '.format(target_query))[0]['cartodb_ids'] + return cartodb_ids, np.concatenate(results) -def fetch_model(model_name): - """ - fetch a model from storage - """ - data = plpy.execute('select * from models where name={model_name}') - if len(data)==0: - plpy.notice('model not found') - data = data[0] - data['model'] = pickle.load(data['model']) - return data - - -def create_model_table(): - """ - create the model table if requred - """ - plpy.execute(''' - CREATE table IF NOT EXISTS _cdb_models( - name TEXT, - model TEXT, - features TEXT[], - accuracy NUMERIC, - table_name TEXT, - census_table_name TEXT, - method TEXT - )''') - -def save_model(model_name,model,accuracy,table_name, column_name,census_table,geoid_column,method): - """ - save a model to the model table for later use - """ - create_model_table() - - plpy.execute(''' - DELETE FROM _cdb_models WHERE name = '{model_name}' - '''.format(**locals())) - - # stringio = StringIO.StringIO() - # gzip_file = gzip.GzipFile(fileobj=stringio, mode='w') - # gzip_file.write() - # gzip_file.close() - - model_pickle = cPickle.dumps(model) #stringio.getvalue() - - - # stringio.close() - - plpy.notice(type(model_pickle)) - plpy.notice(len(model_pickle)) - plpy.notice(sys.getsizeof(model_pickle)) - model_pickle =plpy.quote_literal(model_pickle) - plpy.execute(""" - INSERT INTO _cdb_models VALUES ('{model_name}',$${model_pickle}$$, Array['test1', 'test2'],{accuracy}, '{table_name}', '{census_table}', '{method}') - """.format(**locals())) From 1912d57891d539a15da5df8511f184e8e0c20771 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 21 Jun 2016 17:31:17 -0400 Subject: [PATCH 093/183] replacing dict with ordered dict --- .../crankshaft/crankshaft/clustering/moran.py | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 39b3ff6..103670f 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -7,6 +7,7 @@ Moran's I geostatistics (global clustering & outliers presence) import pysal as ps import plpy +from collections import OrderedDict # crankshaft module import crankshaft.pysal_utils as pu @@ -21,11 +22,11 @@ def moran(subquery, attr_name, core clusters with PySAL. Andy Eschbacher """ - qvals = {"id_col": id_col, - "attr1": attr_name, - "geom_col": geom_col, - "subquery": subquery, - "num_ngbrs": num_ngbrs} + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr_name), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) query = pu.construct_neighbor_query(w_type, qvals) @@ -65,11 +66,11 @@ def moran_local(subquery, attr, # geometries with attributes that are null are ignored # resulting in a collection of not as near neighbors - qvals = {"id_col": id_col, - "attr1": attr, - "geom_col": geom_col, - "subquery": subquery, - "num_ngbrs": num_ngbrs} + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr_name), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) query = pu.construct_neighbor_query(w_type, qvals) @@ -101,12 +102,12 @@ def moran_rate(subquery, numerator, denominator, Moran's I Rate (global) Andy Eschbacher """ - qvals = {"id_col": id_col, - "attr1": numerator, - "attr2": denominator, - "geom_col": geom_col, - "subquery": subquery, - "num_ngbrs": num_ngbrs} + qvals = OrderedDict([("id_col", id_col), + ("attr1", numerator), + ("attr2", denominator) + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) query = pu.construct_neighbor_query(w_type, qvals) @@ -145,13 +146,14 @@ def moran_local_rate(subquery, numerator, denominator, # geometries with values that are null are ignored # resulting in a collection of not as near neighbors - query = pu.construct_neighbor_query(w_type, - {"id_col": id_col, - "numerator": numerator, - "denominator": denominator, - "geom_col": geom_col, - "subquery": subquery, - "num_ngbrs": num_ngbrs}) + qvals = OrderedDict([("id_col", id_col), + ("numerator", numerator), + ("denominator", denominator), + ("geom_col": geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) try: result = plpy.execute(query) @@ -186,12 +188,12 @@ def moran_local_bv(subquery, attr1, attr2, """ plpy.notice('** Constructing query') - qvals = {"num_ngbrs": num_ngbrs, - "attr1": attr1, - "attr2": attr2, - "subquery": subquery, - "geom_col": geom_col, - "id_col": id_col} + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr1), + ("attr2", attr2), + ("geom_col": geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) query = pu.construct_neighbor_query(w_type, qvals) From 7c4314a4113baf852e260af4995e77ca8f8a4a9e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 21 Jun 2016 17:38:49 -0400 Subject: [PATCH 094/183] fix tuple colon --- src/py/crankshaft/crankshaft/clustering/moran.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 103670f..08fe127 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -149,7 +149,7 @@ def moran_local_rate(subquery, numerator, denominator, qvals = OrderedDict([("id_col", id_col), ("numerator", numerator), ("denominator", denominator), - ("geom_col": geom_col), + ("geom_col", geom_col), ("subquery", subquery), ("num_ngbrs", num_ngbrs)]) @@ -191,7 +191,7 @@ def moran_local_bv(subquery, attr1, attr2, qvals = OrderedDict([("id_col", id_col), ("attr1", attr1), ("attr2", attr2), - ("geom_col": geom_col), + ("geom_col", geom_col), ("subquery", subquery), ("num_ngbrs", num_ngbrs)]) From b62d7b32efdb5c96f642a4947fee12f51dd489b9 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 21 Jun 2016 17:41:52 -0400 Subject: [PATCH 095/183] fix variable name --- src/py/crankshaft/crankshaft/clustering/moran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 08fe127..4bced89 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -67,7 +67,7 @@ def moran_local(subquery, attr, # resulting in a collection of not as near neighbors qvals = OrderedDict([("id_col", id_col), - ("attr1", attr_name), + ("attr1", attr), ("geom_col", geom_col), ("subquery", subquery), ("num_ngbrs", num_ngbrs)]) From 4df8257377da0936d3139f5a5f12c492196f5656 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 22 Jun 2016 15:56:47 +0000 Subject: [PATCH 096/183] updating to support passing model paramters and returning accuracy from the function along with prediction --- src/pg/sql/05_segmentation.sql | 15 +++++++++++---- .../crankshaft/segmentation/segmentation.py | 15 ++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 7def90d..3979ca4 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -2,11 +2,18 @@ CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( query TEXT, variable_name TEXT, - target_table TEXT + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1 + ) -RETURNS TABLE (cartodb_id text, prediction Numeric ) +RETURNS TABLE (cartodb_id text, prediction Numeric,accuracy Numeric ) AS $$ from crankshaft.segmentation import create_and_predict_segment - # TODO: use named parameters or a dictionary - return create_and_predict_segment(query,variable_name,target_table) + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + return create_and_predict_segment(query,variable_name,target_table, model_params) $$ LANGUAGE plpythonu; + diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 3881733..d3b327b 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -5,7 +5,7 @@ Segmentation creation and prediction import sklearn import numpy as np import plpy -from sklearn.ensemble import GradientBoostingClassifier +from sklearn.ensemble import GradientBoostingRegressor from sklearn import metrics from sklearn.cross_validation import train_test_split @@ -26,9 +26,10 @@ def get_data(variable, feature_columns, query): )) target = np.array(data[0]['target']) features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) + return replace_nan_with_mean(target), replace_nan_with_mean(features) -def create_and_predict_segment(query,variable,target_query): +def create_and_predict_segment(query,variable,target_query,model_params): """ generate a segment with machine learning Stuart Lynn @@ -38,14 +39,14 @@ def create_and_predict_segment(query,variable,target_query): feature_columns = set(columns) - set([variable, 'the_geom', 'the_geom_webmercator']) target,features = get_data(variable, feature_columns, query) - model, accuracy = train_model(target,features, test_split=0.2) + model, accuracy = train_model(target,features, model_params, 0.2) cartodb_ids, result = predict_segment(model,feature_columns,target_query) - return zip(cartodb_ids, result) + return zip(cartodb_ids, result, np.full(result.shape, accuracy )) -def train_model(target,features,test_split): +def train_model(target,features,model_params,test_split): features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) - model = GradientBoostingClassifier(n_estimators = 200, max_features=features.shape[1]) + model = GradientBoostingRegressor(**model_params) plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) plpy.notice('model trained') @@ -54,7 +55,7 @@ def train_model(target,features,test_split): def calculate_model_accuracy(model,features,target): prediction = model.predict(features) - return metrics.mean_squared_error(prediction,target)/np.std(target) + return metrics.mean_squared_error(prediction,target) def predict_segment(model,features,target_query): """ From 81d7af9e9aeaaedda2e12b3e454d61b0a28286f3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 22 Jun 2016 15:21:09 -0400 Subject: [PATCH 097/183] fixes return problem --- src/pg/sql/08_interpolation.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pg/sql/08_interpolation.sql b/src/pg/sql/08_interpolation.sql index 04f1584..76fad01 100644 --- a/src/pg/sql/08_interpolation.sql +++ b/src/pg/sql/08_interpolation.sql @@ -14,9 +14,12 @@ $$ DECLARE gs geometry[]; vs numeric[]; + output numeric; BEGIN EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs; - RETURN QUERY SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) FROM a; + SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a; + + RETURN output; END; $$ language plpgsql IMMUTABLE; From 1d7f62fa8542a689f532cec5533698d3d6bed1bb Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 22 Jun 2016 20:21:37 +0000 Subject: [PATCH 098/183] adding python tests --- .../crankshaft/segmentation/segmentation.py | 6 +- src/py/crankshaft/test/test_segmentation.py | 69 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 src/py/crankshaft/test/test_segmentation.py diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index d3b327b..9ba4c7c 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -19,7 +19,7 @@ def replace_nan_with_mean(array): def get_data(variable, feature_columns, query): columns = ','.join(['array_agg("{col}") as "{col}"'.format(col=col) for col in feature_columns]) - data = plpy.execute(''' select array_agg("{variable}") as target, {columns} from ({query}) as a'''.format( + data = plpy.execute('''select array_agg("{variable}") as target, {columns} from ({query}) as a'''.format( variable = variable, columns = columns, query = query @@ -34,9 +34,10 @@ def create_and_predict_segment(query,variable,target_query,model_params): generate a segment with machine learning Stuart Lynn """ + columns = plpy.execute('select * from ({query}) a limit 1 '.format(query=query))[0].keys() - feature_columns = set(columns) - set([variable, 'the_geom', 'the_geom_webmercator']) + feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) target,features = get_data(variable, feature_columns, query) model, accuracy = train_model(target,features, model_params, 0.2) @@ -75,7 +76,6 @@ def predict_segment(model,features,target_query): while True: rows = cursor.fetch(batch_size) - if not rows: break batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) diff --git a/src/py/crankshaft/test/test_segmentation.py b/src/py/crankshaft/test/test_segmentation.py new file mode 100644 index 0000000..63abdf6 --- /dev/null +++ b/src/py/crankshaft/test/test_segmentation.py @@ -0,0 +1,69 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.segmentation as segmentation +import json + +class SegmentationTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + + def generate_random_data(self,n_samples,random_state, row_type=False): + x1 = random_state.uniform(size=n_samples) + x2 = random_state.uniform(size=n_samples) + x3 = random_state.randint(0, 4, size=n_samples) + + y = x1+x2*x2+x3 + cartodb_id = range(len(x1)) + + if row_type: + return [ {'features': vals} for vals in zip(x1,x2,x3)], y + else: + return [dict( zip(['x1','x2','x3','target', 'cartodb_id'],[x1,x2,x3,y,cartodb_id]))] + + def test_create_and_predict_segment(self): + n_samples = 1000 + + random_state_train = np.random.RandomState(13) + random_state_test = np.random.RandomState(134) + training_data = self.generate_random_data(n_samples, random_state_train) + test_data, test_y = self.generate_random_data(n_samples, random_state_test, row_type=True) + + ids = [{'cartodb_ids': range(len(test_data))}] + rows = [{'x1': 0,'x2':0,'x3':0,'y':0,'cartodb_id':0}] + + plpy._define_result('select \* from \(select \* from training\) a limit 1',rows) + plpy._define_result('.*from \(select \* from training\) as a' ,training_data) + plpy._define_result('select array_agg\(cartodb\_id order by cartodb\_id\) as cartodb_ids from \(.*\) a',ids) + plpy._define_result('.*select \* from test.*' ,test_data) + + + model_parameters = {'n_estimators': 1200, + 'max_depth': 3, + 'subsample' : 0.5, + 'learning_rate': 0.01, + 'min_samples_leaf': 1} + + result = segmentation.create_and_predict_segment( + 'select * from training', + 'y', + 'select * from test', + model_parameters) + + prediction = [r[1] for r in result] + + accuracy =np.sqrt(np.mean( np.square( np.array(prediction) - np.array(test_y)))) + + self.assertEqual(len(result),len(test_data)) + self.assertTrue( result[0][2] < 0.01) + self.assertTrue( accuracy < 0.5*np.mean(test_y) ) From 89c47dcef6fcf4a8f3f56c6419632add85895995 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 22 Jun 2016 20:21:52 +0000 Subject: [PATCH 099/183] mocking out cursor --- src/py/crankshaft/test/mock_plpy.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py index 63c88f6..c849ec4 100644 --- a/src/py/crankshaft/test/mock_plpy.py +++ b/src/py/crankshaft/test/mock_plpy.py @@ -1,5 +1,16 @@ import re +class MockCursor: + def __init__(self, data): + self.cursor_pos =0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos : self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + class MockPlPy: def __init__(self): self._reset() @@ -27,6 +38,10 @@ class MockPlPy: def info(self, msg): self.infos.append(msg) + def cursor(self,query): + data = self.execute(query) + return MockCursor(data) + def execute(self, query): # TODO: additional arguments for result in self.results: if result[0].match(query): From 6f72075999b3d3887018d9574e4e25abd5214873 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 22 Jun 2016 16:50:10 -0400 Subject: [PATCH 100/183] altering test outputs for less formatting --- src/pg/test/expected/08_interpolation_test.out | 6 ++---- src/pg/test/sql/08_interpolation_test.sql | 5 ++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pg/test/expected/08_interpolation_test.out b/src/pg/test/expected/08_interpolation_test.out index 42d24cb..b927f63 100644 --- a/src/pg/test/expected/08_interpolation_test.out +++ b/src/pg/test/expected/08_interpolation_test.out @@ -1,4 +1,2 @@ - cdb_spatialinterpolation --------------------------- - 780.79470198683925288365 -(1 row) +cdb_spatialinterpolation +t diff --git a/src/pg/test/sql/08_interpolation_test.sql b/src/pg/test/sql/08_interpolation_test.sql index c8db89d..43e7ee9 100644 --- a/src/pg/test/sql/08_interpolation_test.sql +++ b/src/pg/test/sql/08_interpolation_test.sql @@ -1,6 +1,9 @@ +\pset format unaligned +\set ECHO all + WITH a AS ( SELECT ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS vals, ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g ) -SELECT CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'),1) FROM a; +SELECT (cdb_crankshaft.CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'), 1) - 780.79470198683925288365) / 780.79470198683925288365 < 0.001 FROM a; From 6a9045ba62551e4db3637ad99acad52d591092bd Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 22 Jun 2016 16:56:35 -0400 Subject: [PATCH 101/183] updating test outputs --- src/pg/test/expected/08_interpolation_test.out | 8 ++++++++ src/pg/test/sql/08_interpolation_test.sql | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/pg/test/expected/08_interpolation_test.out b/src/pg/test/expected/08_interpolation_test.out index b927f63..49566db 100644 --- a/src/pg/test/expected/08_interpolation_test.out +++ b/src/pg/test/expected/08_interpolation_test.out @@ -1,2 +1,10 @@ +\pset format unaligned +\set ECHO all + +WITH a AS ( + SELECT + ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS vals, + ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g +) cdb_spatialinterpolation t diff --git a/src/pg/test/sql/08_interpolation_test.sql b/src/pg/test/sql/08_interpolation_test.sql index 43e7ee9..ba8968f 100644 --- a/src/pg/test/sql/08_interpolation_test.sql +++ b/src/pg/test/sql/08_interpolation_test.sql @@ -6,4 +6,4 @@ WITH a AS ( ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS vals, ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g ) -SELECT (cdb_crankshaft.CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'), 1) - 780.79470198683925288365) / 780.79470198683925288365 < 0.001 FROM a; +SELECT (cdb_crankshaft.CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'), 1) - 780.79470198683925288365) / 780.79470198683925288365 < 0.001 As cdb_spatialinterpolation FROM a; From 3f210c2a71b02b5b8b6a527b79514a9c19260ce3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 22 Jun 2016 17:08:50 -0400 Subject: [PATCH 102/183] reducing amt of text in outputs --- src/pg/test/expected/08_interpolation_test.out | 10 ++-------- src/pg/test/sql/08_interpolation_test.sql | 3 ++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/pg/test/expected/08_interpolation_test.out b/src/pg/test/expected/08_interpolation_test.out index 49566db..bb8c73e 100644 --- a/src/pg/test/expected/08_interpolation_test.out +++ b/src/pg/test/expected/08_interpolation_test.out @@ -1,10 +1,4 @@ -\pset format unaligned -\set ECHO all - -WITH a AS ( - SELECT - ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS vals, - ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g -) +SET client_min_messages TO WARNING; +\set ECHO none cdb_spatialinterpolation t diff --git a/src/pg/test/sql/08_interpolation_test.sql b/src/pg/test/sql/08_interpolation_test.sql index ba8968f..bd9c729 100644 --- a/src/pg/test/sql/08_interpolation_test.sql +++ b/src/pg/test/sql/08_interpolation_test.sql @@ -1,5 +1,6 @@ +SET client_min_messages TO WARNING; +\set ECHO none \pset format unaligned -\set ECHO all WITH a AS ( SELECT From 2fa087bb62544cda84931f8c39233074572c9564 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 22 Jun 2016 17:11:51 -0400 Subject: [PATCH 103/183] adding row info :/ --- src/pg/test/expected/08_interpolation_test.out | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pg/test/expected/08_interpolation_test.out b/src/pg/test/expected/08_interpolation_test.out index bb8c73e..635ca2a 100644 --- a/src/pg/test/expected/08_interpolation_test.out +++ b/src/pg/test/expected/08_interpolation_test.out @@ -2,3 +2,4 @@ SET client_min_messages TO WARNING; \set ECHO none cdb_spatialinterpolation t +(1 row) From de274bf6284f21f626774826abab7f43dbb9012e Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 22 Jun 2016 21:43:03 +0000 Subject: [PATCH 104/183] removing commented code --- src/py/crankshaft/test/test_segmentation.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/py/crankshaft/test/test_segmentation.py b/src/py/crankshaft/test/test_segmentation.py index 63abdf6..2fc0a2d 100644 --- a/src/py/crankshaft/test/test_segmentation.py +++ b/src/py/crankshaft/test/test_segmentation.py @@ -1,14 +1,6 @@ import unittest import numpy as np - - -# from mock_plpy import MockPlPy -# plpy = MockPlPy() -# -# import sys -# sys.modules['plpy'] = plpy from helper import plpy, fixture_file - import crankshaft.segmentation as segmentation import json From 76b3a873b805d3b3678fe8ff3c5094c421f54f03 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 22 Jun 2016 21:43:32 +0000 Subject: [PATCH 105/183] segmentation pg tests --- src/pg/test/expected/06_segmentation_test.out | 30 + src/pg/test/fixtures/ml_values.sql | 2005 +++++++++++++++++ src/pg/test/sql/06_segmentation_test.sql | 5 + 3 files changed, 2040 insertions(+) create mode 100644 src/pg/test/expected/06_segmentation_test.out create mode 100644 src/pg/test/fixtures/ml_values.sql create mode 100644 src/pg/test/sql/06_segmentation_test.sql diff --git a/src/pg/test/expected/06_segmentation_test.out b/src/pg/test/expected/06_segmentation_test.out new file mode 100644 index 0000000..aa8944a --- /dev/null +++ b/src/pg/test/expected/06_segmentation_test.out @@ -0,0 +1,30 @@ +\pset format unaligned +\set ECHO all +\i test/fixtures/ml_values.sql +SET client_min_messages TO WARNING; +\set ECHO none +_cdb_random_seeds + +(1 row) +prediction +4.5656517130822492 +1.7928053473230694 +1.0283378773916563 +2.6586517814904593 +2.9699056242935944 +3.9550646059951347 +4.1662572444459745 +3.8126334839264162 +1.8809821053623488 +1.6349065129019873 +3.0391288591472954 +3.3035970359672553 +1.5835471589451968 +3.7530378537263638 +1.0833589653009252 +3.8104965452882897 +2.665217959294802 +1.5850334252802472 +3.679401198805563 +3.5332033186588636 +(20 rows) diff --git a/src/pg/test/fixtures/ml_values.sql b/src/pg/test/fixtures/ml_values.sql new file mode 100644 index 0000000..c87a10f --- /dev/null +++ b/src/pg/test/fixtures/ml_values.sql @@ -0,0 +1,2005 @@ +SET client_min_messages TO WARNING; +\set ECHO none +CREATE TABLE ml_values (cartodb_id integer, target float, the_geom geometry, x1 float , x2 float, x3 float, class text); +INSERT INTO ml_values(cartodb_id, target,x1,x2,x3, class) VALUES +(0,1.24382137034,0.811403626309,0.657584780869,0,'train'), +(1,1.72727475342,0.447764244847,0.528687533966,1,'train'), +(2,3.32104694099,0.62774565606,0.832647155118,2,'train'), +(3,3.95282364134,0.881898806954,0.266317168772,3,'train'), +(4,3.80247130968,0.665074747038,0.370670423211,3,'train'), +(5,2.10381192188,0.883366590314,0.469516061027,1,'train'), +(6,1.17893557213,0.711670077404,0.683568207806,0,'train'), +(7,1.80674380603,0.66527165164,0.376127843149,1,'train'), +(8,0.276910799403,0.12432503742,0.390622275329,0,'train'), +(9,1.04011429426,0.0385633572229,0.0393819379254,1,'train'), +(10,1.32965631694,0.208292255072,0.348373451724,1,'train'), +(11,3.91621773493,0.266859692111,0.805827551541,3,'train'), +(12,0.819422907018,0.141031077681,0.823645451233,0,'train'), +(13,0.86861841075,0.865895751347,0.0521791088729,0,'train'), +(14,3.52651073623,0.299745485848,0.476198750922,3,'train'), +(15,3.60697425252,0.806256087392,0.89482856745,2,'train'), +(16,3.25898808929,0.250608720014,0.0915388948687,3,'train'), +(17,1.66522265308,0.429070089585,0.485955310186,1,'train'), +(18,3.162287269,0.131313802907,0.175992801258,3,'train'), +(19,3.52084351408,0.890659436085,0.793841343087,2,'train'), +(20,2.58431797652,0.825925910476,0.87085708704,1,'train'), +(21,2.36135058874,0.836884033221,0.724200632088,1,'train'), +(22,2.95373698573,0.715546042878,0.488048094811,2,'train'), +(23,4.06612839317,0.350181776217,0.846136287455,3,'train'), +(24,1.71396870328,0.203801924342,0.71425960192,1,'train'), +(25,1.02568147198,0.37388512127,0.807339055608,0,'train'), +(26,2.84381682437,0.830147511036,0.11691583866,2,'train'), +(27,3.65269921585,0.507457283779,0.381106195272,3,'train'), +(28,1.36070847289,0.13136938957,0.478893603336,1,'train'), +(29,0.166643149194,0.158235885087,0.0916911342867,0,'train'), +(30,4.02134208186,0.995768081662,0.159918729994,3,'train'), +(31,1.06182066152,0.9315232293,0.360967356166,0,'train'), +(32,1.88309332845,0.0711466915165,0.901080815981,1,'train'), +(33,1.3106116557,0.145148601676,0.406771500996,1,'train'), +(34,4.08110160149,0.809824588582,0.520842598979,3,'train'), +(35,1.95817443835,0.720172089254,0.487854844289,1,'train'), +(36,3.09445483037,0.462289717081,0.795088116679,2,'train'), +(37,3.9825595986,0.896251729074,0.293782010221,3,'train'), +(38,3.30192844335,0.116150213431,0.431019987837,3,'train'), +(39,1.494298122,0.899654633056,0.771131304607,0,'train'), +(40,2.39157161067,0.0900380728132,0.549120695164,2,'train'), +(41,1.63988698756,0.615331744134,0.156701127709,1,'train'), +(42,2.81254514925,0.792540827545,0.141436634949,2,'train'), +(43,1.76453635553,0.171175181353,0.770299405539,1,'train'), +(44,4.19365739446,0.956470538768,0.487018331989,3,'train'), +(45,4.17724977996,0.689416001736,0.698450984842,3,'train'), +(46,3.44571626758,0.121638801413,0.569278021857,3,'train'), +(47,0.807182622364,0.358783853615,0.669625842354,0,'train'), +(48,3.7363232527,0.190185523211,0.739011318916,3,'train'), +(49,1.86578481285,0.985653593244,0.938153089644,0,'train'), +(50,3.3144357104,0.630893004142,0.826766415775,2,'train'), +(51,1.02755553675,0.847642795972,0.424161220266,0,'train'), +(52,4.52608792887,0.723755345588,0.895730195587,3,'train'), +(53,3.55148802171,0.331736804231,0.468776297904,3,'train'), +(54,4.74137857538,0.911580430367,0.910932568864,3,'train'), +(55,2.13162450741,0.982444321588,0.386238508981,1,'train'), +(56,1.31154833663,0.21649611054,0.308305410407,1,'train'), +(57,1.38663340377,0.371776142608,0.121890365347,1,'train'), +(58,1.99685029785,0.18457064921,0.901265581635,1,'train'), +(59,3.70115259653,0.483173481131,0.466882335707,3,'train'), +(60,4.20176093595,0.465662529454,0.857961774494,3,'train'), +(61,1.7184797125,0.346403723018,0.609980318927,1,'train'), +(62,3.14156465929,0.126578834574,0.122416603103,3,'train'), +(63,1.18710602542,0.125258709523,0.248691205919,1,'train'), +(64,3.13150427478,0.124611730533,0.0830213481477,3,'train'), +(65,2.17689203809,0.70068505942,0.690077516419,1,'train'), +(66,1.18342205287,0.773038174481,0.640612112268,0,'train'), +(67,1.9975132827,0.994622569346,0.0537653546163,1,'train'), +(68,0.432033448975,0.421818049735,0.101071258228,0,'train'), +(69,0.661083575367,0.280197452927,0.617159721984,0,'train'), +(70,3.22520898428,0.328672092283,0.946856320674,2,'train'), +(71,3.45938438304,0.0542900404295,0.636470221309,3,'train'), +(72,2.34118444802,0.835758112414,0.710933425577,1,'train'), +(73,1.39379113863,0.0184347707593,0.612663339748,1,'train'), +(74,2.73864731229,0.707545994952,0.176355655807,2,'train'), +(75,2.45820282065,0.552259848697,0.951810365543,1,'train'), +(76,2.58033950412,0.0331182580413,0.739744040921,2,'train'), +(77,0.310831960497,0.25330953236,0.239838337504,0,'train'), +(78,2.8502591452,0.840003315424,0.101271070764,2,'train'), +(79,4.05058044205,0.467543162702,0.763568778404,3,'train'), +(80,0.976286667289,0.776077091797,0.447447846673,0,'train'), +(81,3.79106063676,0.710208952573,0.284344305711,3,'train'), +(82,1.0419324905,0.447613188073,0.770921074055,0,'train'), +(83,3.30507844095,0.669803310816,0.797041485828,2,'train'), +(84,0.421245261502,0.0797069093013,0.584412826862,0,'train'), +(85,2.36278003018,0.67380980013,0.830042306182,1,'train'), +(86,2.5676448622,0.728200751562,0.916211826292,1,'train'), +(87,1.54096382777,0.958926642927,0.762913615585,0,'train'), +(88,3.87941054679,0.71383264077,0.406912651582,3,'train'), +(89,1.3714934845,0.739373292283,0.795059867062,0,'train'), +(90,2.11791863693,0.997242892611,0.347384145181,1,'train'), +(91,1.83165883949,0.774846387789,0.238353627404,1,'train'), +(92,3.93892077639,0.110313686527,0.910278578164,3,'train'), +(93,0.46853056251,0.460889170503,0.087415055953,0,'train'), +(94,2.16415058019,0.790359655066,0.611384433172,1,'train'), +(95,3.19504017985,0.0194446630378,0.419041187492,3,'train'), +(96,1.05622112802,0.729589180698,0.571517232742,0,'train'), +(97,1.45680483768,0.41739274432,0.198524792802,1,'train'), +(98,4.03277024539,0.342001341974,0.831125082894,3,'train'), +(99,3.19597374514,0.136152364733,0.244584096794,3,'train'), +(100,0.674198979717,0.141247074788,0.730035550456,0,'train'), +(101,1.6030553627,0.449039378135,0.392448703099,1,'train'), +(102,3.14484686869,0.422000710067,0.850203598337,2,'train'), +(103,2.93201375767,0.621183774587,0.557521284873,2,'train'), +(104,1.92255465696,0.0239666590258,0.947938815503,1,'train'), +(105,0.750374417884,0.498514838544,0.501856134106,0,'train'), +(106,2.66183539,0.334440387094,0.572184413373,2,'train'), +(107,1.50762714512,0.449439358706,0.241221446846,1,'train'), +(108,4.02466160506,0.989637668904,0.187146830464,3,'train'), +(109,3.9611061532,0.999626040389,0.980550923111,2,'train'), +(110,1.24886880389,0.760865649238,0.698572225794,0,'train'), +(111,3.69094389192,0.158127308309,0.729942863252,3,'train'), +(112,4.47144774067,0.566340168609,0.951371416464,3,'train'), +(113,3.84989330608,0.488585062607,0.601089214236,3,'train'), +(114,3.90544176126,0.333414678464,0.756324720474,3,'train'), +(115,3.60658729206,0.542168232709,0.253809100212,3,'train'), +(116,2.80910993644,0.98391434046,0.90840277189,1,'train'), +(117,0.529897517275,0.37845063724,0.389161765896,0,'train'), +(118,3.33158531793,0.172039843893,0.399431438462,3,'train'), +(119,4.08360113059,0.632897538088,0.67134461531,3,'train'), +(120,0.36058924894,0.130904507554,0.479254359799,0,'train'), +(121,0.948394575974,0.874257647434,0.272280973519,0,'train'), +(122,3.02851043918,0.5276474372,0.70771675265,2,'train'), +(123,0.649506349365,0.144414854231,0.710697893013,0,'train'), +(124,3.86111624,0.0753604101866,0.886428694152,3,'train'), +(125,2.24716868088,0.973162166433,0.5234563157,1,'train'), +(126,1.95686980382,0.956760054448,0.0104761333011,1,'train'), +(127,3.05292710244,0.0385433904369,0.119932114149,3,'train'), +(128,2.62313724007,0.952837884549,0.818718117256,1,'train'), +(129,3.73871514167,0.710940230426,0.16665806684,3,'train'), +(130,1.63871472777,0.33635764867,0.549870056556,1,'train'), +(131,0.325099905771,0.184044265052,0.375573748708,0,'train'), +(132,2.11620242739,0.112128848688,0.0638245932143,2,'train'), +(133,0.1525548278,0.151989744456,0.0237714817413,0,'train'), +(134,3.14701565035,0.998058126006,0.385950157845,2,'train'), +(135,3.64472496462,0.644207576462,0.0227461680587,3,'train'), +(136,1.4811667713,0.432870913159,0.2197631865,1,'train'), +(137,0.486812051883,0.44065685385,0.214837608517,0,'train'), +(138,2.52640998137,0.523932372796,0.0497755821311,2,'train'), +(139,3.58537193033,0.799598306898,0.886438730782,2,'train'), +(140,3.41697465653,0.366453771497,0.224768514327,3,'train'), +(141,1.68831002325,0.66346637256,0.157618687637,1,'train'), +(142,2.50832573117,0.440842333094,0.25977566876,2,'train'), +(143,3.34293652186,0.644421725216,0.835771976464,2,'train'), +(144,1.52687970225,0.455824019792,0.266562717682,1,'train'), +(145,3.38355056147,0.936893026941,0.668324423112,2,'train'), +(146,2.9186681931,0.683796057327,0.484636085919,2,'train'), +(147,3.43119285958,0.35825012194,0.270079132188,3,'train'), +(148,2.44982622143,0.531373877867,0.958359193394,1,'train'), +(149,1.56467859084,0.562383273711,0.0479094680276,1,'train'), +(150,1.4147566619,0.346784386037,0.260714932186,1,'train'), +(151,2.64807483346,0.101312694159,0.739433661189,2,'train'), +(152,0.355070783716,0.330671039063,0.156204176171,0,'train'), +(153,1.63481376891,0.545070698823,0.299571477422,1,'train'), +(154,1.7223421552,0.699505451379,0.151118178335,1,'train'), +(155,4.54679308537,0.635100879441,0.954825746369,3,'train'), +(156,1.97215514937,0.867886595262,0.322906416958,1,'train'), +(157,1.21902077771,0.788118404039,0.656431545303,0,'train'), +(158,3.44049935676,0.749514849173,0.831254778985,2,'train'), +(159,1.00582794774,0.975372732143,0.17451422749,0,'train'), +(160,4.30589162814,0.722820935848,0.763590657549,3,'train'), +(161,0.906132042553,0.894978072798,0.105612356072,0,'train'), +(162,3.22766306171,0.204157590595,0.153314940931,3,'train'), +(163,1.56350600184,0.454133352705,0.33071535969,1,'train'), +(164,2.64217591247,0.726104057929,0.957116426846,1,'train'), +(165,1.60856928053,0.786138835293,0.906879509765,0,'train'), +(166,4.02384662579,0.492593147827,0.728871372714,3,'train'), +(167,3.69385120323,0.58902541735,0.323768105101,3,'train'), +(168,2.04441852581,0.875400416322,0.411118121089,1,'train'), +(169,2.39459275688,0.46138814577,0.966025160704,1,'train'), +(170,1.64761351845,0.497112314903,0.387944846006,1,'train'), +(171,2.68492168842,0.225544125445,0.677773976321,2,'train'), +(172,2.24044390692,0.205190156911,0.187759820009,2,'train'), +(173,0.236505988436,0.167122987329,0.263406532013,0,'train'), +(174,1.69740019481,0.64342808251,0.232318988243,1,'train'), +(175,1.42768204741,0.921523736652,0.711448037991,0,'train'), +(176,2.65464841219,0.410330229053,0.494285527947,2,'train'), +(177,0.220625300247,0.218783719264,0.0429136456521,0,'train'), +(178,0.902080497188,0.875728329944,0.162333506226,0,'train'), +(179,2.3734585601,0.373385586057,0.0085424848658,2,'train'), +(180,2.68418222177,0.498319457905,0.431118039365,2,'train'), +(181,1.65090360296,0.0582627105736,0.76983172992,1,'train'), +(182,2.70121647756,0.700836969263,0.0194809727219,2,'train'), +(183,3.64471767929,0.620811580137,0.154615973154,3,'train'), +(184,0.697598964398,0.35720278891,0.583434808259,0,'train'), +(185,2.09418732317,0.700734656898,0.627258053969,1,'train'), +(186,2.77565652503,0.534894620472,0.490674947966,2,'train'), +(187,2.99118156734,0.944799107597,0.215365874127,2,'train'), +(188,2.16246763706,0.985465980607,0.420715647983,1,'train'), +(189,0.792489856319,0.0470340536513,0.863397824104,0,'train'), +(190,0.687119116403,0.150185800385,0.732757337744,0,'train'), +(191,4.00045366032,0.772811734907,0.477118355766,3,'train'), +(192,4.00531986539,0.890961861059,0.338168603402,3,'train'), +(193,3.98787982655,0.917170421429,0.265912401226,3,'train'), +(194,0.800103167701,0.583836264582,0.465045054934,0,'train'), +(195,3.16136367592,0.157590380223,0.0614271576693,3,'train'), +(196,1.24032152242,0.222620887834,0.133043731843,1,'train'), +(197,3.94586252163,0.936739896945,0.0955124321134,3,'train'), +(198,0.724612971022,0.556639623191,0.409845517031,0,'train'), +(199,1.71914971653,0.486382998421,0.482459032575,1,'train'), +(200,0.701027027956,0.42292722954,0.527351683809,0,'train'), +(201,1.00451594424,0.213906286204,0.88916233503,0,'train'), +(202,1.28560098048,0.458749446911,0.909313770693,0,'train'), +(203,4.18130653509,0.872826563929,0.555409732681,3,'train'), +(204,3.70513653927,0.718920678846,0.993084014786,2,'train'), +(205,0.883517177383,0.836370992358,0.217131722751,0,'train'), +(206,2.41822052154,0.397240247753,0.144845689579,2,'train'), +(207,3.6881507522,0.134094496362,0.744349552182,3,'train'), +(208,2.94061893438,0.819973365795,0.347340709657,2,'train'), +(209,1.96736985525,0.932919137643,0.185609045059,1,'train'), +(210,3.55671188827,0.522939874975,0.183771633548,3,'train'), +(211,2.59491479328,0.569803336867,0.158465947167,2,'train'), +(212,0.845317716043,0.760524441834,0.291192847111,0,'train'), +(213,3.84865511262,0.953008053247,0.946386316139,2,'train'), +(214,1.79421500329,0.719882898171,0.272639148184,1,'train'), +(215,1.61283368743,0.580804023591,0.178968331944,1,'train'), +(216,1.69166880922,0.994233556172,0.835125890536,0,'train'), +(217,3.6663085942,0.558387950098,0.328512776159,3,'train'), +(218,4.14739895574,0.42903354328,0.847564400183,3,'train'), +(219,3.84834184259,0.829160865001,0.13849540639,3,'train'), +(220,3.70418573269,0.703884379256,0.0173595343695,3,'train'), +(221,2.32691481998,0.85344048504,0.68809471364,1,'train'), +(222,1.12861991727,0.639123203252,0.699640417657,0,'train'), +(223,3.3697979033,0.411858825745,0.978743621974,2,'train'), +(224,2.38363760199,0.38363455315,0.00174609204425,2,'train'), +(225,4.12587571793,0.945570023524,0.424624180199,3,'train'), +(226,1.64781419808,0.953412012914,0.833307977382,0,'train'), +(227,3.88605908504,0.743467379141,0.377613169665,3,'train'), +(228,3.85260504053,0.200479368566,0.807542984592,3,'train'), +(229,0.620538868538,0.5957157602,0.157553509444,0,'train'), +(230,2.96759001799,0.906294428571,0.247579460818,2,'train'), +(231,0.803879749102,0.359243472208,0.666810525482,0,'train'), +(232,1.21289265217,0.736679873124,0.690081719108,0,'train'), +(233,2.69090231294,0.624244952275,0.258180868115,2,'train'), +(234,2.06499182801,0.0439160520786,0.145174983831,2,'train'), +(235,1.91462241156,0.491150554019,0.65074715331,1,'train'), +(236,3.36652154268,0.53627297948,0.911179764482,2,'train'), +(237,0.984331075591,0.9624499175,0.14792281126,0,'train'), +(238,0.985365737701,0.683660823188,0.549276719435,0,'train'), +(239,0.480626074803,0.422862762119,0.240339993935,0,'train'), +(240,3.89794584279,0.0764720388433,0.906351920586,3,'train'), +(241,1.41850659471,0.396382097208,0.148743058659,1,'train'), +(242,2.76083817204,0.785713543335,0.987483989086,1,'train'), +(243,0.630662008259,0.0476826574982,0.763530844669,0,'train'), +(244,1.63198293695,0.630147700114,0.0428396642867,1,'train'), +(245,3.66212785948,0.0821554006103,0.761559228736,3,'train'), +(246,2.69689387455,0.210582393177,0.697360366938,2,'train'), +(247,3.83034962216,0.959005393505,0.93345820938,2,'train'), +(248,0.525457665501,0.524263722989,0.0345534732191,0,'train'), +(249,0.861481042856,0.162623542445,0.835976973613,0,'train'), +(250,1.18923074552,0.13365024941,0.235755161358,1,'train'), +(251,2.97758839764,0.395334103472,0.7630558919,2,'train'), +(252,1.94085977346,0.935959937907,0.0699988253456,1,'train'), +(253,2.27729296713,0.236608890328,0.201702941988,2,'train'), +(254,1.16865293427,0.262844805223,0.951739527942,0,'train'), +(255,0.876256070676,0.599286264016,0.526279209793,0,'train'), +(256,3.4820175379,0.637430802843,0.919014001558,2,'train'), +(257,2.62263559323,0.214770605542,0.638643083172,2,'train'), +(258,4.26774699502,0.678896999789,0.767365620305,3,'train'), +(259,1.92742959729,0.038076073495,0.943055419259,1,'train'), +(260,0.830310249039,0.336972363075,0.702380157724,0,'train'), +(261,0.190093286729,0.189345746521,0.027341181537,0,'train'), +(262,1.85856149645,0.350967827008,0.712456082468,1,'train'), +(263,2.19868876763,0.794310568466,0.635907382534,1,'train'), +(264,2.57775279682,0.282887761421,0.54301476536,2,'train'), +(265,3.20347370976,0.633154837581,0.75519459226,2,'train'), +(266,0.632469198504,0.270791595383,0.601396377709,0,'train'), +(267,0.922686896791,0.849793576252,0.269987630345,0,'train'), +(268,1.17592252313,0.793421991901,0.618466273316,0,'train'), +(269,1.16516284095,0.578767949852,0.765764252949,0,'train'), +(270,1.01311122453,0.888906154698,0.352427396545,0,'train'), +(271,2.65175162691,0.0944527639096,0.746524522709,2,'train'), +(272,1.45375342664,0.874605879696,0.761017441943,0,'train'), +(273,3.47583088126,0.253728465861,0.471277429331,3,'train'), +(274,0.589399547981,0.258841119105,0.574942109152,0,'train'), +(275,3.86372858515,0.891783117826,0.985872946849,2,'train'), +(276,1.06581404933,0.0593192118575,0.0805905545103,1,'train'), +(277,1.1873391595,0.0866856156474,0.317259426729,1,'train'), +(278,0.865600889401,0.0958516230905,0.877353558328,0,'train'), +(279,1.31207151088,0.99504370054,0.563052227007,0,'train'), +(280,3.27474268384,0.0126251383396,0.511974164877,3,'train'), +(281,1.47102201088,0.461570924759,0.0972166967251,1,'train'), +(282,2.29687679698,0.842379621346,0.674164056911,1,'train'), +(283,0.844185133675,0.275029507464,0.754424036077,0,'train'), +(284,2.07111091001,0.381008814631,0.830723838214,1,'train'), +(285,0.077539650653,0.0763769730851,0.0340980581244,0,'train'), +(286,1.8276930277,0.0952086879959,0.85585298954,1,'train'), +(287,4.23841977745,0.559781482165,0.823795056602,3,'train'), +(288,0.815109004853,0.341787559485,0.687983608357,0,'train'), +(289,3.46743270302,0.160450701172,0.55405956525,3,'train'), +(290,2.42266450219,0.961978440732,0.678738581085,1,'train'), +(291,2.99004689776,0.956935036248,0.18196664946,2,'train'), +(292,2.71323856164,0.692224597576,0.144961940055,2,'train'), +(293,2.05482704476,0.582246228265,0.68744513708,1,'train'), +(294,3.72152683971,0.665255175512,0.237216492248,3,'train'), +(295,1.81261247465,0.290164799032,0.722805420299,1,'train'), +(296,2.95071568645,0.301091012724,0.805992973746,2,'train'), +(297,3.60095700472,0.590015080655,0.104603652264,3,'train'), +(298,3.22976379154,0.170499829501,0.24344190691,3,'train'), +(299,2.53052994672,0.400046262715,0.361225253824,2,'train'), +(300,4.29248168564,0.714346239362,0.760352185683,3,'train'), +(301,3.63419301177,0.730822756503,0.950457918724,2,'train'), +(302,0.838103142117,0.736571494019,0.318640311477,0,'train'), +(303,2.31953053783,0.314612236636,0.0701306009595,2,'train'), +(304,2.63166163285,0.179836930722,0.67217906999,2,'train'), +(305,0.975304021572,0.296559450289,0.823859557985,0,'train'), +(306,1.75094033332,0.31909260818,0.657151219389,1,'train'), +(307,0.664783393542,0.169172615206,0.703996291422,0,'train'), +(308,2.04218742681,0.784692561315,0.507439519049,1,'train'), +(309,1.57636682723,0.383858484348,0.438757726866,1,'train'), +(310,1.2282706945,0.146782608227,0.285461181734,1,'train'), +(311,0.551187755918,0.144170303427,0.637979194404,0,'train'), +(312,2.56058324309,0.483713500532,0.277253931547,2,'train'), +(313,4.15764790974,0.958622990947,0.446122089557,3,'train'), +(314,2.79121629685,0.689782775695,0.318486296655,2,'train'), +(315,0.748431598562,0.137843689504,0.781401247156,0,'train'), +(316,1.66420987831,0.250719037897,0.64303253449,1,'train'), +(317,2.29475888839,0.0881384447859,0.454555215132,2,'train'), +(318,3.74719508348,0.236978994873,0.714294119121,3,'train'), +(319,2.78454298817,0.678070344718,0.326301460998,2,'train'), +(320,1.10371480513,0.0796255420211,0.155207161917,1,'train'), +(321,1.76641748403,0.863825876329,0.950048213354,0,'train'), +(322,2.49335475767,0.316214332717,0.42088053525,2,'train'), +(323,3.54333632187,0.172684341126,0.608811942016,3,'train'), +(324,2.22166069926,0.193473509034,0.167890411361,2,'train'), +(325,2.12581391522,0.117233470914,0.0926306876841,2,'train'), +(326,1.1106964371,0.8863218799,0.473681915635,0,'train'), +(327,2.15757621852,0.173216092038,0.992149246075,1,'train'), +(328,2.73658434122,0.358163646105,0.615159081143,2,'train'), +(329,2.59452938859,0.567128328148,0.165532656726,2,'train'), +(330,0.204381212023,0.145453208602,0.242750908178,0,'train'), +(331,2.566539304,0.0338857225982,0.729831200622,2,'train'), +(332,1.50047183779,0.150044407722,0.59196911243,1,'train'), +(333,1.20318271888,0.60453212288,0.773725142411,0,'train'), +(334,4.14756975063,0.997470494993,0.38742645191,3,'train'), +(335,2.54704842308,0.964634439528,0.763160522797,1,'train'), +(336,3.49783786504,0.987582572101,0.7143215613,2,'train'), +(337,2.46569832119,0.98128027012,0.696001473467,1,'train'), +(338,3.00370483816,0.801531906037,0.449636444387,2,'train'), +(339,3.34946013288,0.170840493359,0.422634167475,3,'train'), +(340,1.8134430762,0.76012023851,0.230917382812,1,'train'), +(341,4.01785111498,0.848006557061,0.412122018243,3,'train'), +(342,2.38847231817,0.156840694177,0.481281231702,2,'train'), +(343,4.21071602043,0.896237445362,0.560783893375,3,'train'), +(344,2.76188152894,0.194672158523,0.753133036332,2,'train'), +(345,1.39793034843,0.460185425697,0.968372305848,0,'train'), +(346,2.95426195892,0.0959496686049,0.926451450598,2,'train'), +(347,3.86987063976,0.868526979401,0.036655973104,3,'train'), +(348,0.563491456666,0.266468990277,0.544997675581,0,'train'), +(349,2.85618091544,0.855777109473,0.0200949238374,2,'train'), +(350,1.91344572093,0.725274473213,0.433787099525,1,'train'), +(351,3.56002399866,0.677283328575,0.939542798431,2,'train'), +(352,0.791124753553,0.769353271193,0.147551626086,0,'train'), +(353,1.30748850069,0.0113869481294,0.544152141007,1,'train'), +(354,1.51803181516,0.581141627833,0.967930879414,0,'train'), +(355,3.56002976317,0.142404802346,0.646239089516,3,'train'), +(356,0.596999512076,0.596434559096,0.0237687395507,0,'train'), +(357,2.10739518646,0.882980331155,0.473724450818,1,'train'), +(358,0.799164494823,0.788047867806,0.105435416331,0,'train'), +(359,1.64987834729,0.523003187071,0.356195396126,1,'train'), +(360,2.16786155357,0.150961121726,0.13000166092,2,'train'), +(361,3.61040687165,0.659965886629,0.974905628775,2,'train'), +(362,2.97545681733,0.982196849344,0.996624286269,1,'train'), +(363,4.29316606884,0.844843787562,0.669568727825,3,'train'), +(364,4.44648332608,0.835021494275,0.781960249507,3,'train'), +(365,2.25057031703,0.000773777407512,0.499796498213,2,'train'), +(366,1.25257530532,0.0909098245332,0.402076461366,1,'train'), +(367,3.67135736022,0.8949240159,0.881154551891,2,'train'), +(368,4.41622174539,0.664159254405,0.867215365977,3,'train'), +(369,2.18574155326,0.492151046784,0.832820812945,1,'train'), +(370,1.9143141796,0.868178728805,0.21479164508,1,'train'), +(371,2.6496939551,0.381294704864,0.518072630275,2,'train'), +(372,3.80102066555,0.149213176068,0.807345954028,3,'train'), +(373,2.7772675577,0.873441815607,0.950697502938,1,'train'), +(374,2.66383064066,0.210515315051,0.673286956364,2,'train'), +(375,1.18744758619,0.806568574132,0.61715396139,0,'train'), +(376,2.50432137301,0.504193972627,0.0112871777609,2,'train'), +(377,1.65232178342,0.190782640275,0.679366722135,1,'train'), +(378,0.216548289434,0.0350450971435,0.426031914639,0,'train'), +(379,1.89801398531,0.854429884034,0.208768056165,1,'train'), +(380,2.59459973115,0.498244828348,0.310410861276,2,'train'), +(381,1.49906884919,0.887433584663,0.782071137768,0,'train'), +(382,3.83949120738,0.776244945907,0.2514880941,3,'train'), +(383,0.996948415073,0.945722943258,0.226330448272,0,'train'), +(384,2.23674163998,0.934670617374,0.549609882198,1,'train'), +(385,3.86753529951,0.439466120143,0.65426995909,3,'train'), +(386,2.75218469685,0.266691198166,0.696773635184,2,'train'), +(387,3.67226558137,0.0359568909104,0.797689595305,3,'train'), +(388,1.18396975781,0.78847940941,0.628880233748,0,'train'), +(389,1.8281691901,0.636934297672,0.437304118921,1,'train'), +(390,3.39316302573,0.231497124561,0.402076984127,3,'train'), +(391,1.97156837706,0.810477779969,0.401360931191,1,'train'), +(392,0.326005646879,0.31542799544,0.102847709934,0,'train'), +(393,3.77529987287,0.193138447597,0.762995036207,3,'train'), +(394,0.404991080326,0.26462005965,0.374661207862,0,'train'), +(395,2.54244371023,0.246605747233,0.543909885,2,'train'), +(396,2.6684293602,0.549178196702,0.345327617626,2,'train'), +(397,1.34663375311,0.305396180334,0.203070364109,1,'train'), +(398,3.03821883597,0.718848856216,0.565128286107,2,'train'), +(399,3.30844448365,0.197294045161,0.333392319178,3,'train'), +(400,1.76708778971,0.351660952206,0.644536141346,1,'train'), +(401,2.19288933129,0.166187481641,0.163407006113,2,'train'), +(402,1.03244020978,0.030791397522,0.0406055693155,1,'train'), +(403,0.771574541234,0.444845689166,0.571602004955,0,'train'), +(404,3.79033185346,0.734240049438,0.236837083289,3,'train'), +(405,1.14478684595,0.102240151174,0.206268501653,1,'train'), +(406,3.40874657815,0.0865458594342,0.567627270941,3,'train'), +(407,1.06491450952,0.982979841473,0.286242323995,0,'train'), +(408,0.535434837721,0.418233058076,0.342347454561,0,'train'), +(409,2.41173077867,0.701821898754,0.842560905762,1,'train'), +(410,4.09289201995,0.544201347332,0.740736574377,3,'train'), +(411,3.44256370804,0.749248071362,0.83265577322,2,'train'), +(412,3.11232634313,0.0652220974382,0.217035125477,3,'train'), +(413,1.43287829588,0.970202680338,0.680202628298,0,'train'), +(414,3.74688543887,0.456671085777,0.538715465803,3,'train'), +(415,1.52774773516,0.40919367052,0.344316808541,1,'train'), +(416,3.51592792958,0.0347311982555,0.693683451816,3,'train'), +(417,1.15781412817,0.820426435015,0.580850835546,0,'train'), +(418,0.469889852652,0.436650793301,0.182315823095,0,'train'), +(419,2.58749479728,0.0534933012757,0.730754059861,2,'train'), +(420,4.1065431692,0.834869600156,0.521223147072,3,'train'), +(421,0.809284017219,0.524846417573,0.533326916296,0,'train'), +(422,2.58595893036,0.553485629255,0.180203499146,2,'train'), +(423,2.25777140465,0.72624304772,0.729059913125,1,'train'), +(424,1.72917055396,0.710125073451,0.13800536406,1,'train'), +(425,0.124649936754,0.124115071305,0.0231271582434,0,'train'), +(426,1.53509666829,0.998531920468,0.732505800537,0,'train'), +(427,3.02847113374,0.79440790934,0.483800810667,2,'train'), +(428,4.3851179319,0.783297574467,0.775770815016,3,'train'), +(429,3.27087178027,0.100197546736,0.413127381732,3,'train'), +(430,3.22844173875,0.835680707857,0.626706495012,2,'train'), +(431,2.74104331843,0.373032611942,0.606638860025,2,'train'), +(432,1.95049168742,0.752227208753,0.445268995854,1,'train'), +(433,1.29644698132,0.471302832527,0.908374454061,0,'train'), +(434,3.93537153239,0.733225365863,0.44960667981,3,'train'), +(435,3.81501278203,0.728951014591,0.293362859677,3,'train'), +(436,4.10866710473,0.207987014993,0.949041669127,3,'train'), +(437,2.7634739237,0.590973913842,0.415331205009,2,'train'), +(438,1.53152308102,0.117656420726,0.643324692742,1,'train'), +(439,0.652380533996,0.319926263427,0.576588475925,0,'train'), +(440,4.2112267977,0.762487307135,0.66988020613,3,'train'), +(441,1.62726294874,0.159706593681,0.683780926218,1,'train'), +(442,4.10543372071,0.223663519748,0.939026198232,3,'train'), +(443,2.90859925322,0.437374419854,0.686458180349,2,'train'), +(444,1.1195055957,0.998546959599,0.347791081106,0,'train'), +(445,1.16528637542,0.0825892555745,0.2875710692,1,'train'), +(446,2.69981869457,0.394716282032,0.552360763032,2,'train'), +(447,2.31472640151,0.466597210444,0.920939298255,1,'train'), +(448,0.840007432044,0.839287186984,0.0268373817632,0,'train'), +(449,3.37533076432,0.53220518824,0.918218697303,2,'train'), +(450,0.498196246143,0.494527456286,0.0605705362125,0,'train'), +(451,3.63867107652,0.604844737758,0.183919381144,3,'train'), +(452,2.44169783629,0.209551918468,0.48181523204,2,'train'), +(453,3.90584318551,0.905412754464,0.0207468321212,3,'train'), +(454,2.20274681323,0.317611550398,0.940816274751,1,'train'), +(455,1.07374375619,0.045673775221,0.167540982968,1,'train'), +(456,2.22789450557,0.406208007853,0.906469248081,1,'train'), +(457,0.738025792896,0.591130044355,0.383269811675,0,'train'), +(458,4.37758286643,0.403908240559,0.986749525398,3,'train'), +(459,4.18993686556,0.504331421886,0.82801294898,3,'train'), +(460,2.36744693265,0.291644989671,0.275321526539,2,'train'), +(461,4.27484787332,0.822147391766,0.672830202619,3,'train'), +(462,2.62893385742,0.0381880076456,0.768599928296,2,'train'), +(463,2.82103680329,0.267611552273,0.743925568196,2,'train'), +(464,3.33576384176,0.014899312033,0.566449053076,3,'train'), +(465,0.494069350277,0.424832132736,0.263129659181,0,'train'), +(466,4.33546872608,0.449962442518,0.941013434313,3,'train'), +(467,3.1657740975,0.164681194777,0.0330590792866,3,'train'), +(468,0.913594110102,0.106448948744,0.898412578584,0,'train'), +(469,0.0876071701603,0.0667651930665,0.144367507057,0,'train'), +(470,1.01627143078,0.972344146897,0.209588367727,0,'train'), +(471,2.99194992775,0.949094558881,0.207015383182,2,'train'), +(472,1.78094705038,0.255829485466,0.724649960263,1,'train'), +(473,3.54883347116,0.129841055614,0.647296234767,3,'train'), +(474,2.03002797557,0.959341658726,0.265868984358,1,'train'), +(475,3.29448636962,0.11285360726,0.426183953659,3,'train'), +(476,2.18606596886,0.129753823782,0.237301801664,2,'train'), +(477,3.5107261593,0.510192916321,0.0230920544285,3,'train'), +(478,1.15017973632,0.78358600181,0.605469846076,0,'train'), +(479,2.76224893245,0.738760418257,0.153259630026,2,'train'), +(480,2.31756997885,0.910168791138,0.638279866288,1,'train'), +(481,2.76038513132,0.982550038137,0.881949597868,1,'train'), +(482,4.01748906583,0.83510152367,0.427068545034,3,'train'), +(483,2.12787166797,0.372905103465,0.868888119669,1,'train'), +(484,2.57957368103,0.667245129304,0.955158914385,1,'train'), +(485,3.70415295095,0.495616433551,0.456657987337,3,'train'), +(486,1.64592760512,0.278527801341,0.606135136566,1,'train'), +(487,3.83709283289,0.641371840859,0.442403652818,3,'train'), +(488,2.56129870357,0.426296283084,0.367426755263,2,'train'), +(489,3.91279276709,0.934702510826,0.988984457039,2,'train'), +(490,4.20220011238,0.919698696644,0.531508622448,3,'train'), +(491,4.38469431267,0.490125275978,0.945816597811,3,'train'), +(492,1.56277743791,0.504768673882,0.240850086209,1,'train'), +(493,1.50934492129,0.619605815082,0.943259829639,0,'train'), +(494,2.18043863997,0.995838278886,0.429651441389,1,'train'), +(495,4.50005282664,0.825362772495,0.821395187561,3,'train'), +(496,0.959664747905,0.295857333997,0.814743771936,0,'train'), +(497,2.02829946359,0.00093705357447,0.165415869914,2,'train'), +(498,3.09189531627,0.747972936298,0.586448957684,2,'train'), +(499,0.431017371436,0.335701245336,0.308733098485,0,'train'), +(500,1.10440839723,0.552078386709,0.743189081273,0,'train'), +(501,0.945440898882,0.547411936875,0.63089536534,0,'train'), +(502,3.21316616106,0.1278379616,0.292109909896,3,'train'), +(503,1.84119994845,0.678165806943,0.403774864881,1,'train'), +(504,4.15718744193,0.693794586114,0.680729649579,3,'train'), +(505,0.692883161304,0.0272184267333,0.815882794628,0,'train'), +(506,1.71919116925,0.286770639048,0.657586899357,1,'train'), +(507,2.06057940992,0.642684745845,0.646447727255,1,'train'), +(508,0.734500135299,0.591537050228,0.378104595411,0,'train'), +(509,2.35886483681,0.355858321504,0.0548316998636,2,'train'), +(510,0.420694453626,0.007326589821,0.642936904996,0,'train'), +(511,2.86294289087,0.931893954384,0.964908771069,1,'train'), +(512,3.58832352341,0.349095720459,0.489109193284,3,'train'), +(513,2.85911785279,0.291534969988,0.753380967905,2,'train'), +(514,0.846780074256,0.680312260217,0.408004674041,0,'train'), +(515,3.30972718486,0.983222183524,0.571406161447,2,'train'), +(516,1.61022366452,0.582124087192,0.167629285416,1,'train'), +(517,1.46214580154,0.512692054312,0.974399172426,0,'train'), +(518,3.36788507401,0.300056049787,0.260440058799,3,'train'), +(519,3.86622353876,0.671197074989,0.441618006616,3,'train'), +(520,1.30602576682,0.034902869059,0.520694630053,1,'train'), +(521,1.3973559051,0.382415653242,0.122230322993,1,'train'), +(522,3.04765684433,0.0473962605573,0.0161426074989,3,'train'), +(523,1.53295789351,0.947557649786,0.765114529806,0,'train'), +(524,1.35895602915,0.74174013481,0.785630889377,0,'train'), +(525,3.79107471836,0.663054385947,0.357799290676,3,'train'), +(526,0.682890680332,0.68049614357,0.04893400415,0,'train'), +(527,2.68560513419,0.666283334171,0.139002877753,2,'train'), +(528,1.5138392269,0.52363126972,0.995091934035,0,'train'), +(529,2.29955146416,0.23688463329,0.250333439373,2,'train'), +(530,1.59246339626,0.228447971637,0.603336908059,1,'train'), +(531,3.64118876073,0.328684609331,0.559020707489,3,'train'), +(532,3.92236220744,0.89379113153,0.169029807748,3,'train'), +(533,3.02840062853,0.0167931789879,0.107737874204,3,'train'), +(534,3.05651013525,0.143665482385,0.95542904125,2,'train'), +(535,2.24584169722,0.320794807793,0.961793579427,1,'train'), +(536,2.4364261646,0.993502324935,0.66552523593,1,'train'), +(537,2.68993308109,0.446820334697,0.493064647277,2,'train'), +(538,4.3362925093,0.687250501777,0.805631434045,3,'train'), +(539,1.49375392384,0.372570856679,0.348113583706,1,'train'), +(540,3.82351336106,0.294574528184,0.727281811181,3,'train'), +(541,1.04094657631,0.277220124312,0.873914442034,0,'train'), +(542,2.31074431435,0.164069798301,0.38298109098,2,'train'), +(543,0.387592305402,0.353433661517,0.184820572137,0,'train'), +(544,1.30577223408,0.725768091129,0.761580030562,0,'train'), +(545,1.05124211258,0.326339499993,0.851412128518,0,'train'), +(546,1.16293518244,0.794492210847,0.606995034238,0,'train'), +(547,1.24059722178,0.237315759746,0.0572840469566,1,'train'), +(548,4.24004273423,0.670864672017,0.754438905556,3,'train'), +(549,1.94584105524,0.937247982949,0.0926988257174,1,'train'), +(550,0.906581414292,0.627344492022,0.528428729603,0,'train'), +(551,0.582423039372,0.577423399341,0.0707081327041,0,'train'), +(552,3.1744387083,0.976620911041,0.444767127,2,'train'), +(553,0.730856941869,0.454536615883,0.525661798104,0,'train'), +(554,2.33401498768,0.00919969253603,0.569925692655,2,'train'), +(555,3.31951115713,0.674448745998,0.803157774741,2,'train'), +(556,3.91313155555,0.469991735382,0.665687479353,3,'train'), +(557,2.2622878505,0.802162974088,0.67832505218,1,'train'), +(558,0.746195614582,0.439817436143,0.553514388647,0,'train'), +(559,1.39984555259,0.0984201123667,0.549022258407,1,'train'), +(560,3.06781140178,0.0161174641057,0.227363008578,3,'train'), +(561,3.06977099155,0.647397815513,0.649902435782,2,'train'), +(562,2.15437747815,0.0108508940941,0.378849025417,2,'train'), +(563,3.93255222969,0.906042385669,0.16281843881,3,'train'), +(564,1.52093726409,0.293162475924,0.477257570048,1,'train'), +(565,0.241088946455,0.189777524898,0.226520245359,0,'train'), +(566,1.22449638039,0.856499693862,0.606627304471,0,'train'), +(567,0.468568241538,0.457405975709,0.105651624827,0,'train'), +(568,2.14640700567,0.145085132675,0.0363575714233,2,'train'), +(569,1.82693783152,0.812944360193,0.118294003784,1,'train'), +(570,1.2685894001,0.173071530841,0.309059653232,1,'train'), +(571,1.85904737527,0.437692943801,0.6491181953,1,'train'), +(572,1.96354731002,0.194727840668,0.87682351095,1,'train'), +(573,2.17849619334,0.177828197415,0.025845617141,2,'train'), +(574,0.771941573176,0.736889769663,0.187221268859,0,'train'), +(575,3.16306686795,0.275838911088,0.941927787497,2,'train'), +(576,3.05990093593,0.998452693121,0.247887560811,2,'train'), +(577,2.11155741709,0.828257469462,0.532259286086,1,'train'), +(578,3.62719426631,0.116995754165,0.714281815631,3,'train'), +(579,0.771270150397,0.540639803068,0.480239885193,0,'train'), +(580,2.89355871232,0.663866396152,0.479262262408,2,'train'), +(581,1.66476389871,0.627070017213,0.194149121811,1,'train'), +(582,1.92564463876,0.353884908039,0.756147955575,1,'train'), +(583,3.06666120309,0.27155798321,0.891685605963,2,'train'), +(584,4.42952712424,0.818864583972,0.781449000427,3,'train'), +(585,0.753397387605,0.355818165957,0.630538834369,0,'train'), +(586,1.41371889414,0.396968172406,0.129424579343,1,'train'), +(587,2.0839171445,0.856548372034,0.476832017032,1,'train'), +(588,3.17150965715,0.318092789183,0.923805644046,2,'train'), +(589,1.4895290775,0.95476029651,0.731278866777,0,'train'), +(590,3.36473547392,0.490875136745,0.934804972802,2,'train'), +(591,2.99221426911,0.901284255985,0.301546038153,2,'train'), +(592,1.27960206088,0.0682052722718,0.459779065,1,'train'), +(593,3.38536561206,0.377904399177,0.0863783125523,3,'train'), +(594,0.644814164502,0.522337991631,0.349965959589,0,'train'), +(595,3.9654577968,0.812431656871,0.391185556898,3,'train'), +(596,1.9254902965,0.830963898738,0.307451455951,1,'train'), +(597,1.97701211335,0.883130426583,0.306401185967,1,'train'), +(598,3.4736588422,0.248167532959,0.474859252037,3,'train'), +(599,1.48362948566,0.539573277645,0.971625549281,0,'train'), +(600,3.3592534876,0.179340764215,0.424161199768,3,'train'), +(601,2.07128477674,0.0349609893738,0.190588004251,2,'train'), +(602,3.29067149956,0.432722131743,0.92625556291,2,'train'), +(603,1.5344838351,0.692619201057,0.917531816368,0,'train'), +(604,1.74572177974,0.285404197809,0.67846708242,1,'train'), +(605,3.9530290139,0.974574370864,0.989168662584,2,'train'), +(606,3.5518908479,0.602663068128,0.974283213326,2,'train'), +(607,4.9241038632,0.949510976834,0.98721471138,3,'train'), +(608,4.05888901245,0.207640753461,0.922631160861,3,'train'), +(609,3.5709155848,0.797356220433,0.879522236424,2,'train'), +(610,1.69327291681,0.138219735528,0.745018913372,1,'train'), +(611,2.31989251562,0.607549074802,0.844004408054,1,'train'), +(612,1.86762731819,0.616936454164,0.500690387391,1,'train'), +(613,1.38906829047,0.191578211182,0.444398559057,1,'train'), +(614,1.3504785947,0.308199348624,0.205619177313,1,'train'), +(615,0.782505977699,0.727772286602,0.233952326546,0,'train'), +(616,2.60291214904,0.0132415326267,0.767900134398,2,'train'), +(617,0.956542546629,0.231905032946,0.851256432389,0,'train'), +(618,1.01480229765,0.58596709406,0.654855101214,0,'train'), +(619,2.88052105394,0.837993350939,0.206222459987,2,'train'), +(620,1.97527237647,0.960567079291,0.121265399753,1,'train'), +(621,2.26801322237,0.200779342141,0.259294967618,2,'train'), +(622,2.37994972022,0.351892338036,0.167503379615,2,'train'), +(623,2.84054373779,0.823412755683,0.130885377737,2,'train'), +(624,3.88791650362,0.0559361677106,0.912129560925,3,'train'), +(625,1.00400302511,0.446928498162,0.746374253944,0,'train'), +(626,2.33488666829,0.257172000037,0.278773507084,2,'train'), +(627,0.893105656942,0.849557331986,0.208682354205,0,'train'), +(628,3.03238168333,0.00299493230483,0.171425642841,3,'train'), +(629,1.10807643915,0.649692903311,0.677040276378,0,'train'), +(630,2.26582203114,0.226634857035,0.197957505803,2,'train'), +(631,2.5443345758,0.171366693497,0.610710964617,2,'train'), +(632,1.49535083799,0.495134590279,0.0147053633082,1,'train'), +(633,4.28454765733,0.816343290208,0.684254606944,3,'train'), +(634,1.03194179244,0.409810231167,0.78875316879,0,'train'), +(635,3.64228965597,0.536740158877,0.32488382091,3,'train'), +(636,1.13005581937,0.11941513339,0.103153700781,1,'train'), +(637,3.1520887166,0.96198819152,0.436005189276,2,'train'), +(638,0.666405440181,0.103248254724,0.750437995744,0,'train'), +(639,3.30912819412,0.152626190352,0.395603341452,3,'train'), +(640,0.745269331126,0.742901399851,0.0486613940959,0,'train'), +(641,1.90255200165,0.897306888368,0.0724231542902,1,'train'), +(642,1.03137999187,0.238531157703,0.890420593968,0,'train'), +(643,2.59524544441,0.275224104957,0.565704286221,2,'train'), +(644,3.08882766013,0.155026540735,0.966333855038,2,'train'), +(645,0.534129987832,0.0340213072181,0.70718362581,0,'train'), +(646,2.37679211278,0.203334853448,0.416482003613,2,'train'), +(647,3.45486746732,0.376971875643,0.279097817392,3,'train'), +(648,2.76543758993,0.230114384821,0.731657847022,2,'train'), +(649,1.02591751794,0.951688544567,0.272449946541,0,'train'), +(650,0.359810570836,0.0800404459204,0.528933006075,0,'train'), +(651,0.486987204846,0.249618036951,0.487205467842,0,'train'), +(652,1.80869661167,0.511377476515,0.545269781991,1,'train'), +(653,1.45399626469,0.316677444251,0.37056554136,1,'train'), +(654,3.00908726103,0.987469441926,0.147029993904,2,'train'), +(655,3.88378285476,0.88318300657,0.0244917983512,3,'train'), +(656,2.06684700599,0.315941249135,0.866548184956,1,'train'), +(657,3.91911290956,0.108302168878,0.900450298842,3,'train'), +(658,1.57945249851,0.46517358655,0.338051640969,1,'train'), +(659,3.49677332069,0.371367013936,0.354127528941,3,'train'), +(660,2.40887402461,0.903523794162,0.710879898751,1,'train'), +(661,1.21741976737,0.0451816825109,0.415015764594,1,'train'), +(662,2.61079837947,0.972815709163,0.798738173816,1,'train'), +(663,3.84863884749,0.595770240395,0.502860425062,3,'train'), +(664,4.1374973138,0.717960727672,0.647716439597,3,'train'), +(665,2.55413334541,0.354314219794,0.447011326048,2,'train'), +(666,3.15647917932,0.869355115022,0.535839588211,2,'train'), +(667,1.17540927044,0.848011368526,0.572186946649,0,'train'), +(668,3.7415117792,0.816983132261,0.961524127072,2,'train'), +(669,3.68092030094,0.422499956808,0.508350611423,3,'train'), +(670,4.09798509889,0.705172088493,0.626747964013,3,'train'), +(671,0.722843358899,0.70617283972,0.129114364728,0,'train'), +(672,2.26556095716,0.930885697276,0.578511244387,1,'train'), +(673,2.85536438423,0.115947263352,0.859893668356,2,'train'), +(674,2.81607612378,0.74679673614,0.263209778767,2,'train'), +(675,1.36041377427,0.82540534443,0.731442704414,0,'train'), +(676,1.45759032149,0.147613684927,0.556755454903,1,'train'), +(677,2.85392551069,0.0727477746501,0.883842596866,2,'train'), +(678,3.31010759849,0.289122295372,0.144863049541,3,'train'), +(679,4.14334510553,0.92676282048,0.465384018906,3,'train'), +(680,3.48824310645,0.444564223256,0.208994935806,3,'train'), +(681,3.21491520598,0.701082007727,0.716821594436,2,'train'), +(682,0.724320902947,0.678767084014,0.213433406318,0,'train'), +(683,3.62177654488,0.899057752117,0.850128691882,2,'train'), +(684,2.10614898564,0.730593443211,0.612825866319,1,'train'), +(685,2.64537470486,0.653620476998,0.995868579616,1,'train'), +(686,0.215018532823,0.105306520581,0.331228036619,0,'train'), +(687,0.505516921954,0.379845580399,0.354501539566,0,'train'), +(688,1.82150462465,0.304048850706,0.719343988609,1,'train'), +(689,2.71847901632,0.358078703512,0.600333501319,2,'train'), +(690,4.11912569662,0.508067936207,0.781701836004,3,'train'), +(691,3.00897261869,0.25898246576,0.866019718555,2,'train'), +(692,0.782397713346,0.774348740496,0.0897160679632,0,'train'), +(693,0.938680550391,0.332386546162,0.77864883242,0,'train'), +(694,3.84065188299,0.0338781170005,0.898205859471,3,'train'), +(695,1.49121482249,0.858120050764,0.795672527943,0,'train'), +(696,2.10470668542,0.171925802401,0.965805820554,1,'train'), +(697,3.37281145241,0.661115820916,0.843620549476,2,'train'), +(698,3.38331967048,0.318783153871,0.254040383821,3,'train'), +(699,1.43129333879,0.163228324172,0.51774995376,1,'train'), +(700,2.52741414884,0.337829134484,0.435413612961,2,'train'), +(701,0.953970684609,0.202030491513,0.867144851277,0,'train'), +(702,0.543237745766,0.172624497115,0.608780131616,0,'train'), +(703,1.32376070114,0.722848761628,0.775185100161,0,'train'), +(704,0.72804023832,0.70565535173,0.149615796593,0,'train'), +(705,1.07860407567,0.953943477068,0.353073078277,0,'train'), +(706,0.898436369629,0.510123882364,0.623147243647,0,'train'), +(707,1.79328573793,0.23390972682,0.747914441037,1,'train'), +(708,1.20546817876,0.535504220336,0.818513260994,0,'train'), +(709,3.25849956264,0.182171722489,0.276274935808,3,'train'), +(710,3.36539973056,0.548944226377,0.903579273877,2,'train'), +(711,2.20768360824,0.221222906304,0.993207280446,1,'train'), +(712,0.763688671927,0.0762037898321,0.829147081099,0,'train'), +(713,2.31428605424,0.265557424375,0.220745622525,2,'train'), +(714,2.23260844404,0.160313071248,0.268877988673,2,'train'), +(715,1.41627690058,0.146814878869,0.519097314302,1,'train'), +(716,0.728696487379,0.510952711751,0.466630234369,0,'train'), +(717,3.56014747125,0.216810503711,0.585949628841,3,'train'), +(718,2.34117788408,0.0989076145477,0.492209578867,2,'train'), +(719,2.49427997317,0.193473445816,0.548458318702,2,'train'), +(720,1.12555716091,0.0922252252327,0.182570358143,1,'train'), +(721,1.10700886834,0.100248842948,0.0822193735651,1,'train'), +(722,2.48658116272,0.272913365175,0.462242141679,2,'train'), +(723,0.681839495212,0.681827802871,0.00341940659408,0,'train'), +(724,1.22270572491,0.21965998946,0.0551881821668,1,'train'), +(725,3.32333977248,0.87222251745,0.671652629737,2,'train'), +(726,2.47792619634,0.507703204666,0.984998980547,1,'train'), +(727,2.98220980684,0.59959737364,0.618556734019,2,'train'), +(728,2.1920753408,0.0083494343666,0.428632600756,2,'train'), +(729,2.85396274929,0.815287788457,0.196659504816,2,'train'), +(730,3.66500000528,0.658318029399,0.0817433537257,3,'train'), +(731,3.53685356519,0.784972026427,0.867111030237,2,'train'), +(732,3.0467056926,0.394635734334,0.807508488046,2,'train'), +(733,2.89203975883,0.86058012132,0.177368648623,2,'train'), +(734,0.431289845628,0.374520598288,0.238262979374,0,'train'), +(735,0.627621983958,0.571522481242,0.236853335877,0,'train'), +(736,1.76222405923,0.786213961527,0.987932233353,0,'train'), +(737,3.90108859988,0.892698590648,0.0915969935484,3,'train'), +(738,1.00824020786,0.809901655648,0.445352166504,0,'train'), +(739,0.573219436278,0.538727422752,0.1857202561,0,'train'), +(740,3.64895115275,0.307117889626,0.584665086285,3,'train'), +(741,0.236858711257,0.0114133571358,0.474810861419,0,'train'), +(742,2.81548278042,0.36006129573,0.67484923108,2,'train'), +(743,1.09889749697,0.0721705405948,0.163483810751,1,'train'), +(744,0.847468119179,0.108478274687,0.859645185232,0,'train'), +(745,3.01132259874,0.197499150101,0.902121637387,2,'train'), +(746,0.473285172799,0.385097036162,0.296964874417,0,'train'), +(747,1.51791738897,0.520437786654,0.998739006105,0,'train'), +(748,0.758146740412,0.741283589849,0.129858194055,0,'train'), +(749,0.813467576678,0.470118431819,0.585960019847,0,'train'), +(750,3.48360379109,0.839676713863,0.802450669652,2,'train'), +(751,2.37934769258,0.373447854177,0.0768104055884,2,'train'), +(752,0.485352842652,0.477219465024,0.0901852406362,0,'train'), +(753,4.56312660095,0.724907255871,0.91554319673,3,'train'), +(754,1.39606247302,0.314830889607,0.285011549607,1,'train'), +(755,1.07584378619,0.877489309046,0.445370045178,0,'train'), +(756,0.348907856813,0.326897100427,0.148360225079,0,'train'), +(757,0.541766102633,0.540661536382,0.0332350154282,0,'train'), +(758,3.00754864175,0.760057833149,0.49748448076,2,'train'), +(759,4.19085275321,0.753373744067,0.661421959983,3,'train'), +(760,3.75657559034,0.755226273152,0.036733053132,3,'train'), +(761,0.958281626211,0.933844468687,0.156323886606,0,'train'), +(762,2.84844220909,0.979059239209,0.932407083779,1,'train'), +(763,0.391096031332,0.0626595064201,0.573093818595,0,'train'), +(764,1.72778752786,0.727601196294,0.013650332081,1,'train'), +(765,1.75991628766,0.346177830203,0.64322504418,1,'train'), +(766,0.217757321845,0.214515467875,0.0569372810196,0,'train'), +(767,0.625162166518,0.103990282929,0.721922352881,0,'train'), +(768,2.99954339415,0.913919185581,0.292616145434,2,'train'), +(769,3.14408357105,0.970139957702,0.417065478493,2,'train'), +(770,4.45027168388,0.704262000876,0.863718520703,3,'train'), +(771,3.08313959596,0.243314613191,0.91641965429,2,'train'), +(772,0.564599092629,0.55497815543,0.0980863762156,0,'train'), +(773,2.03558008962,0.985957378349,0.222761556986,1,'train'), +(774,1.62188046085,0.272763752363,0.59086098914,1,'train'), +(775,2.90384386535,0.89761352196,0.0789325242619,2,'train'), +(776,3.94581453174,0.797879247105,0.38462356224,3,'train'), +(777,3.19184212652,0.0778032298582,0.337696456389,3,'train'), +(778,0.757984587615,0.101319032883,0.81034903266,0,'train'), +(779,3.17387699437,0.173871849162,0.00226830601557,3,'train'), +(780,3.10139259002,0.0378312159584,0.252113811721,3,'train'), +(781,3.49439249576,0.228316275025,0.51582576587,3,'train'), +(782,3.43715460165,0.164838244945,0.521839397421,3,'train'), +(783,4.91025144752,0.939925112887,0.985051437557,3,'train'), +(784,3.16781331966,0.489409175268,0.823652927142,2,'train'), +(785,3.36386096861,0.159963464883,0.45155011209,3,'train'), +(786,1.67673346419,0.547476303398,0.359523519115,1,'train'), +(787,2.7835685288,0.565778319948,0.46667998548,2,'train'), +(788,0.979752741281,0.939332186493,0.201048637867,0,'train'), +(789,2.51346807528,0.417686324694,0.30948626882,2,'train'), +(790,2.2346788067,0.690748410812,0.737516369912,1,'train'), +(791,2.40394378424,0.447863237165,0.977793713967,1,'train'), +(792,2.72654124556,0.671291357908,0.235052946489,2,'train'), +(793,1.08854116424,0.840075139264,0.498463664653,0,'train'), +(794,0.953122843546,0.396684214999,0.745948140655,0,'train'), +(795,3.23608545166,0.0666854960378,0.411582258631,3,'train'), +(796,1.32283272732,0.733806139302,0.76748067599,0,'train'), +(797,1.9722553054,0.957186626784,0.122754546229,1,'train'), +(798,0.627662918881,0.40124992114,0.475828748333,0,'train'), +(799,3.64775453138,0.220636671145,0.653542546616,3,'train'), +(800,2.98838440389,0.878915265432,0.330861207247,2,'train'), +(801,0.54336838096,0.126560701856,0.645606442892,0,'train'), +(802,3.7160641128,0.176307607664,0.734681226883,3,'train'), +(803,1.62344556238,0.0795330022565,0.73750427804,1,'train'), +(804,0.6678325499,0.570964245243,0.311236734107,0,'train'), +(805,0.906791817372,0.87719888792,0.172025955753,0,'train'), +(806,3.14677150065,0.856070899027,0.539166580587,2,'train'), +(807,0.933623237072,0.928531680834,0.0713551416389,0,'train'), +(808,2.46457532225,0.742823121166,0.849560004404,1,'train'), +(809,2.07854298038,0.763236801438,0.561521307645,1,'train'), +(810,1.05945527933,0.0571458100517,0.0480569379203,1,'train'), +(811,2.0753568234,0.328551775511,0.864178828649,1,'train'), +(812,1.4161183371,0.965560948376,0.671235717708,0,'train'), +(813,0.942869512223,0.803607393167,0.373178401111,0,'train'), +(814,0.434189233541,0.398336329103,0.189348631992,0,'train'), +(815,3.16421332872,0.469369158313,0.833573134408,2,'train'), +(816,0.877888510582,0.554817722226,0.568393163537,0,'train'), +(817,0.342256663894,0.0920928100983,0.500163826956,0,'train'), +(818,1.19437033781,0.958987036411,0.485163169873,0,'train'), +(819,2.4185222072,0.866575557093,0.7429311207,1,'train'), +(820,2.29702798937,0.508479232246,0.888002678557,1,'train'), +(821,2.27737701721,0.0426738921994,0.484461685802,2,'train'), +(822,1.13744938714,0.80327605942,0.578077267949,0,'train'), +(823,4.0991696931,0.586477571092,0.716025224419,3,'train'), +(824,1.34348757554,0.117170271196,0.475728183259,1,'train'), +(825,1.30440617617,0.375707588427,0.963690089054,0,'train'), +(826,2.2282483173,0.902165860634,0.571036300651,1,'train'), +(827,1.52656308845,0.5178091057,0.0935627209437,1,'train'), +(828,3.2429930153,0.43859738444,0.896881057255,2,'train'), +(829,3.48827157132,0.307373235648,0.425321449818,3,'train'), +(830,4.02685704625,0.132103837442,0.945913954231,3,'train'), +(831,1.05194933008,0.919265725091,0.364257608006,0,'train'), +(832,0.984672980151,0.952146351326,0.180351403726,0,'train'), +(833,2.95532888049,0.892889765465,0.249878200376,2,'train'), +(834,2.32095170827,0.872783732429,0.669453490424,1,'train'), +(835,3.75167335434,0.777071808185,0.987219097341,2,'train'), +(836,4.43624524123,0.478645228242,0.978570392453,3,'train'), +(837,0.493424617077,0.0498549496803,0.666010260729,0,'train'), +(838,3.9147927984,0.276291640244,0.799062674738,3,'train'), +(839,3.4198464471,0.396590191226,0.152500019265,3,'train'), +(840,1.27720483278,0.190638817152,0.29422103193,1,'train'), +(841,1.580936325,0.227904729719,0.594164619684,1,'train'), +(842,4.09466632998,0.936198268004,0.398080471731,3,'train'), +(843,1.30974316712,0.700471277387,0.780558703582,0,'train'), +(844,4.46191166039,0.881566201107,0.761804081953,3,'train'), +(845,0.0967506298265,0.0586957183066,0.195076681128,0,'train'), +(846,1.29916323675,0.291640023736,0.0867364572546,1,'train'), +(847,3.01170465403,0.944238471623,0.259742531,2,'train'), +(848,1.02612193445,0.652305745793,0.611405093748,0,'train'), +(849,3.39127260388,0.0446467757273,0.588749376346,3,'train'), +(850,2.40974411311,0.406408704746,0.0577529944119,2,'train'), +(851,2.17496825201,0.049944414591,0.353587100192,2,'train'), +(852,0.0449696231403,0.0380253394684,0.0833323686929,0,'train'), +(853,3.10792775531,0.107707836669,0.0148296540516,3,'train'), +(854,3.1009708213,0.571423454077,0.727700053059,2,'train'), +(855,2.66205273838,0.615416134198,0.215955097616,2,'train'), +(856,0.398343314672,0.194725706889,0.451240077767,0,'train'), +(857,3.42989580267,0.11396257019,0.562079382718,3,'train'), +(858,0.872845586455,0.872074811413,0.027762835634,0,'train'), +(859,1.01562878893,0.160300629277,0.924839531841,0,'train'), +(860,2.18750777785,0.614652514205,0.756872025938,1,'train'), +(861,3.6194255144,0.188497116264,0.656451367688,3,'train'), +(862,3.99363244841,0.989165241372,0.0668371681136,3,'train'), +(863,1.09300321982,0.199292331136,0.945362834408,0,'train'), +(864,3.42490120268,0.304694569109,0.346708283107,3,'train'), +(865,2.83813005817,0.769298134294,0.262358388227,2,'train'), +(866,3.22130563731,0.221304610449,0.00101334267253,3,'train'), +(867,2.02549033192,0.0240931491588,0.037378907932,2,'train'), +(868,4.12425029953,0.904176742613,0.469119981362,3,'train'), +(869,0.740432160855,0.0706798250473,0.81838397822,0,'train'), +(870,2.14564666895,0.964014894267,0.426182794922,1,'train'), +(871,3.68866819913,0.132065510642,0.746058099944,3,'train'), +(872,3.77130436563,0.763485018313,0.088427073443,3,'train'), +(873,0.703580918285,0.701962996215,0.0402234020203,0,'train'), +(874,1.83792292411,0.761602930498,0.276260734828,1,'train'), +(875,4.12029105794,0.833128182455,0.535875802297,3,'train'), +(876,2.82952596243,0.603048162809,0.475896837164,2,'train'), +(877,3.53876022216,0.582939761857,0.977660708174,2,'train'), +(878,3.49477215853,0.308148921783,0.431999116609,3,'train'), +(879,3.13048058934,0.575020109959,0.745292210732,2,'train'), +(880,3.1524980097,0.408512608084,0.862545883776,2,'train'), +(881,0.4229523324,0.365511850675,0.239667439851,0,'train'), +(882,2.20605366521,0.856995324245,0.590811595149,1,'train'), +(883,3.4892010668,0.767209072477,0.849701120584,2,'train'), +(884,0.160009943613,0.0916419043668,0.261472826974,0,'train'), +(885,4.71639652384,0.933767676698,0.884663126361,3,'train'), +(886,4.43421482904,0.992118536821,0.664903220189,3,'train'), +(887,3.32658636318,0.964516567288,0.601722357813,2,'train'), +(888,1.78519326735,0.761645439254,0.15345301592,1,'train'), +(889,2.21692608965,0.838661335479,0.615032319617,1,'train'), +(890,3.47516222055,0.744867712785,0.854572704787,2,'train'), +(891,1.52395923661,0.250633958142,0.522805201259,1,'train'), +(892,0.325735256557,0.236160325793,0.299290712793,0,'train'), +(893,2.81791079024,0.701856111712,0.340667988698,2,'train'), +(894,1.48728396068,0.394561566257,0.304503521192,1,'train'), +(895,0.397280198733,0.368601543974,0.169347733257,0,'train'), +(896,1.14982690429,0.948659432186,0.448516969699,0,'train'), +(897,2.22475700667,0.568393496392,0.810162644337,1,'train'), +(898,2.40712295578,0.222796613645,0.429332437785,2,'train'), +(899,3.21131611811,0.168476120319,0.20697825439,3,'train'), +(900,3.71464223054,0.452960586716,0.511548281029,3,'train'), +(901,0.222339523039,0.217049254224,0.0727342341283,0,'train'), +(902,3.99284107273,0.0906436446099,0.949840738292,3,'train'), +(903,1.6694185709,0.652064858526,0.131733489955,1,'train'), +(904,0.72648755406,0.71488655639,0.107707927611,0,'train'), +(905,1.0517420052,0.228148264941,0.907520655554,0,'train'), +(906,2.21657991343,0.844265982491,0.610175328031,1,'train'), +(907,4.02602526832,0.136026052682,0.943397697492,3,'train'), +(908,3.91164726022,0.611059059104,0.548259246264,3,'train'), +(909,4.05249309774,0.134708332182,0.958010837915,3,'train'), +(910,0.317620851239,0.0870424441656,0.480185804739,0,'train'), +(911,0.126737160202,0.105126235306,0.14700654712,0,'train'), +(912,1.07833625758,0.437347966474,0.800617443667,0,'train'), +(913,3.1593651061,0.426484055879,0.856084721403,2,'train'), +(914,3.51496843983,0.765842951482,0.865520356982,2,'train'), +(915,1.37401226506,0.956519981551,0.646136427937,0,'train'), +(916,4.24975475059,0.901133629716,0.590441462695,3,'train'), +(917,3.27510500703,0.313508101999,0.980610475688,2,'train'), +(918,2.87249433045,0.534970619834,0.580967908422,2,'train'), +(919,0.680590696151,0.460698101224,0.468927067812,0,'train'), +(920,4.06249482883,0.961230433943,0.318220670106,3,'train'), +(921,3.5208320043,0.809432340923,0.843445115808,2,'train'), +(922,3.38586140084,0.980409011669,0.636751434367,2,'train'), +(923,1.31270405892,0.199283633024,0.336779491507,1,'train'), +(924,1.08171591786,0.759752751913,0.56741798169,0,'train'), +(925,0.494048874062,0.493051635885,0.0315790781609,0,'train'), +(926,1.9541834616,0.883074702195,0.266662257184,1,'train'), +(927,3.28478298911,0.105363182792,0.423579752014,3,'train'), +(928,3.35596940015,0.198665784439,0.396615198536,3,'train'), +(929,1.43221235797,0.804595418616,0.792222783916,0,'train'), +(930,1.04151871739,0.0205844406083,0.144686823119,1,'train'), +(931,1.5573820282,0.864245592582,0.832548158138,0,'train'), +(932,0.850080657327,0.330153886806,0.721059477797,0,'train'), +(933,2.97850716878,0.619954263612,0.598792873341,2,'train'), +(934,0.57780127819,0.374295514667,0.451116130861,0,'train'), +(935,2.41942331853,0.0901423684633,0.573830070725,2,'train'), +(936,2.32338043422,0.384286695731,0.96906849009,1,'train'), +(937,1.0470787516,0.941459565527,0.324991055371,0,'train'), +(938,1.07923145778,0.317569591178,0.87273241409,0,'train'), +(939,2.33092538397,0.325371126098,0.0745268936308,2,'train'), +(940,2.60973559809,0.558275441315,0.226848312248,2,'train'), +(941,0.301873361397,0.210058481924,0.303009701945,0,'train'), +(942,1.92740318686,0.742962444954,0.42946564695,1,'train'), +(943,1.67324947457,0.673223362835,0.00510996446302,1,'train'), +(944,3.69688198197,0.442128635767,0.504730964177,3,'train'), +(945,1.93806386024,0.326367369678,0.782110280306,1,'train'), +(946,2.92587133413,0.465784473861,0.678297029528,2,'train'), +(947,1.59706726815,0.470594075229,0.355630697386,1,'train'), +(948,4.01042965214,0.781479433591,0.478487427786,3,'train'), +(949,4.34001007846,0.979308099385,0.600584697666,3,'train'), +(950,1.94568194003,0.286976987772,0.811606402303,1,'train'), +(951,2.48297192781,0.0703715354453,0.64233977953,2,'train'), +(952,3.43330486985,0.396866211666,0.190889125366,3,'train'), +(953,1.01049586623,0.759914861982,0.500580667078,0,'train'), +(954,0.606684884966,0.606410996736,0.0165495688711,0,'train'), +(955,4.516391481,0.932148046169,0.764358184903,3,'train'), +(956,1.45346440397,0.801876812536,0.807209756776,0,'train'), +(957,1.68322515944,0.387716339265,0.543607229693,1,'train'), +(958,3.52421319161,0.447168165065,0.277569858861,3,'train'), +(959,4.26284699331,0.688773154009,0.757676606015,3,'train'), +(960,1.20507542159,0.149018099633,0.236764275078,1,'train'), +(961,1.12653097139,0.6108689372,0.718096117658,0,'train'), +(962,2.16176705503,0.117254286354,0.21098049359,2,'train'), +(963,4.01356988893,0.672924387389,0.583648440019,3,'train'), +(964,3.86596494117,0.58173810517,0.533129286385,3,'train'), +(965,3.74667437944,0.544965521271,0.449120093254,3,'train'), +(966,2.74708319026,0.46598963354,0.530182569237,2,'train'), +(967,1.93197539131,0.790156105698,0.3765890142,1,'train'), +(968,3.38527452627,0.735576131329,0.806038705612,2,'train'), +(969,1.65966515441,0.656857808986,0.052984388504,1,'train'), +(970,2.50244248573,0.420062826889,0.287018568802,2,'train'), +(971,0.882958911742,0.37928633108,0.709698936636,0,'train'), +(972,1.4473343414,0.749503416286,0.835362750613,0,'train'), +(973,2.06038545094,0.908171444662,0.390146134519,1,'train'), +(974,0.70032257833,0.631457821076,0.262420954296,0,'train'), +(975,0.824186006975,0.7790660492,0.212414589366,0,'train'), +(976,4.94692718169,0.988679668696,0.978901176318,3,'train'), +(977,4.60296701793,0.865329679505,0.858858159666,3,'train'), +(978,4.50227138876,0.511692946573,0.995278072797,3,'train'), +(979,2.39104720558,0.537323210028,0.923971858635,1,'train'), +(980,3.77180068741,0.726794910228,0.212145650861,3,'train'), +(981,1.5247556069,0.460231751865,0.254015462195,1,'train'), +(982,1.68555720238,0.0739576495766,0.782048305929,1,'train'), +(983,2.91878772737,0.614469635115,0.551650335133,2,'train'), +(984,0.4676761178,0.330730590893,0.370061517734,0,'train'), +(985,2.31800820581,0.58701259216,0.854982814825,1,'train'), +(986,0.528420123049,0.527751394053,0.0258597949753,0,'train'), +(987,3.57257439721,0.569240335338,0.057741335945,3,'train'), +(988,1.49686453769,0.521534051101,0.987588217119,0,'train'), +(989,4.57689441388,0.65460205574,0.960360535496,3,'train'), +(990,2.94633319462,0.161663955217,0.885815578664,2,'train'), +(991,1.5877465381,0.534156901422,0.231494355609,1,'train'), +(992,2.65206272391,0.846418308035,0.897576969332,1,'train'), +(993,3.27111841261,0.244618976334,0.162786474474,3,'train'), +(994,4.09702326597,0.15086983814,0.972704183106,3,'train'), +(995,2.1538807163,0.425956851832,0.853184543033,1,'train'), +(996,3.83328066689,0.83031000015,0.0545038231579,3,'train'), +(997,1.08009466281,0.60012473769,0.692798618012,0,'train'), +(998,4.10648685932,0.70347100304,0.634835298543,3,'train'), +(999,1.43186970739,0.429642285891,0.0471955665007,1,'train'), +(1000,4.61585521712,0.777702410574,0.915506857725,3,'test'), +(1001,1.75829629844,0.237541220035,0.721633617848,1,'test'), +(1002,0.99861844769,0.824278532661,0.417540315453,0,'test'), +(1003,2.66485237446,0.965749198043,0.836123900158,1,'test'), +(1004,2.97260783117,0.972601113905,0.00259176786914,2,'test'), +(1005,3.98054771655,0.453449247417,0.726015474442,3,'test'), +(1006,4.18523414916,0.609042462761,0.759072912438,3,'test'), +(1007,3.8268713481,0.775526514605,0.226593983796,3,'test'), +(1008,1.94121836211,0.641613344759,0.547361870571,1,'test'), +(1009,1.67239363485,0.722018229517,0.97487199433,0,'test'), +(1010,3.03506699441,0.0350365241014,0.0055199916905,3,'test'), +(1011,3.30056773356,0.298449470889,0.0460245876972,3,'test'), +(1012,1.55263976535,0.0585124918821,0.70294187062,1,'test'), +(1013,3.79731586597,0.857060942587,0.969667429269,2,'test'), +(1014,1.10864822417,0.372854027875,0.857784469608,0,'test'), +(1015,3.80429985696,0.679847951578,0.352777416195,3,'test'), +(1016,2.64680289571,0.256279949327,0.62491835177,2,'test'), +(1017,1.57582576814,0.347581215152,0.477749466763,1,'test'), +(1018,3.651056129,0.00941277008097,0.801026440839,3,'test'), +(1019,3.55478111772,0.358333782705,0.443223797888,3,'test'), +(1020,1.35554435873,0.949094181678,0.637534451657,0,'test'), +(1021,0.861500842905,0.217899009132,0.802247987703,0,'test'), +(1022,4.0377106654,0.31939136638,0.847537196246,3,'test'), +(1023,2.15284892186,0.917772386001,0.484846920028,1,'test'), +(1024,1.04039166204,0.0319036664399,0.0921303185799,1,'test'), +(1025,2.42785443478,0.0650845370425,0.602303825105,2,'test'), +(1026,1.50775500547,0.629828999108,0.936977057542,0,'test'), +(1027,1.30984054026,0.87381344328,0.660323479048,0,'test'), +(1028,0.00942572686202,0.00871573230378,0.0266457230758,0,'test'), +(1029,4.30027100952,0.746577236994,0.744106022371,3,'test'), +(1030,1.60641384579,0.812841171003,0.890826961192,0,'test'), +(1031,1.11032930335,0.0757174461723,0.186042621927,1,'test'), +(1032,1.01926933899,0.656455334597,0.602340438952,0,'test'), +(1033,0.636798999146,0.509262200084,0.357122946704,0,'test'), +(1034,4.19933978491,0.479883391496,0.848207753684,3,'test'), +(1035,1.95689678229,0.955574144931,0.036368081527,1,'test'), +(1036,2.2859919064,1.20335694851e-05,0.534770860116,2,'test'), +(1037,1.33057587679,0.246978700992,0.289131762007,1,'test'), +(1038,3.71164928992,0.712232677912,0.999708263449,2,'test'), +(1039,3.32484243213,0.324582049787,0.0161363671181,3,'test'), +(1040,1.34057906726,0.276996356384,0.252156124011,1,'test'), +(1041,2.94037513323,0.695445452553,0.494903708486,2,'test'), +(1042,2.9335306374,0.918551748146,0.122388272526,2,'test'), +(1043,1.42860689936,0.244475702198,0.429105112017,1,'test'), +(1044,0.477310600576,0.458085817277,0.138653464793,0,'test'), +(1045,2.80452510659,0.252992682822,0.742652289953,2,'test'), +(1046,4.23954214907,0.379333291483,0.927474451176,3,'test'), +(1047,0.612695832598,0.604538828632,0.0903161334791,0,'test'), +(1048,2.57152996012,0.772378759681,0.893952571695,1,'test'), +(1049,1.64690797811,0.0679174967944,0.760914240444,1,'test'), +(1050,3.78951570565,0.686085079187,0.321606322179,3,'test'), +(1051,1.09922322062,0.548260097396,0.742268902234,0,'test'), +(1052,1.02307381137,0.137986052758,0.940791028132,0,'test'), +(1053,1.89794283363,0.0987532192049,0.893974056909,1,'test'), +(1054,2.43634085555,0.245559105325,0.436785702862,2,'test'), +(1055,2.45877421392,0.151786662584,0.554064573255,2,'test'), +(1056,2.06008699874,0.92599447949,0.366186454212,1,'test'), +(1057,2.71120700724,0.680105015605,0.176357567561,2,'test'), +(1058,0.413559844959,0.23765892188,0.419405439973,0,'test'), +(1059,2.74834334585,0.568885252557,0.423624944139,2,'test'), +(1060,2.63585770775,0.556632051434,0.281470524771,2,'test'), +(1061,3.36239358484,0.0727372108843,0.538197337377,3,'test'), +(1062,1.40698576923,0.839708510416,0.753178105641,0,'test'), +(1063,2.46385947845,0.405319492698,0.241950378694,2,'test'), +(1064,2.2960820441,0.144870989046,0.388858656909,2,'test'), +(1065,1.87946335431,0.190920058667,0.829785090033,1,'test'), +(1066,3.098338964,0.490640137065,0.779550400507,2,'test'), +(1067,1.28500053528,0.71202437425,0.756951888187,0,'test'), +(1068,2.06369395299,0.984938457701,0.280634095024,1,'test'), +(1069,2.68403288496,0.874786501735,0.899581226584,1,'test'), +(1070,0.593604832636,0.499041683901,0.307511217251,0,'test'), +(1071,1.24313989366,0.106779993994,0.36926941339,1,'test'), +(1072,3.81991947344,0.91321280699,0.952211460995,2,'test'), +(1073,3.11905267979,0.364915960886,0.868410455316,2,'test'), +(1074,3.58367910147,0.226587876899,0.5975711042,3,'test'), +(1075,2.46689941224,0.872431862345,0.771017217638,1,'test'), +(1076,0.141327622382,0.136358351759,0.0704930537199,0,'test'), +(1077,3.20567114734,0.236380160153,0.984525767659,2,'test'), +(1078,2.65720902691,0.595399245352,0.248615730707,2,'test'), +(1079,3.56919073809,0.563922609079,0.0725818779634,3,'test'), +(1080,4.54973724354,0.958934731983,0.768636787798,3,'test'), +(1081,0.983117317815,0.453239333224,0.727927183578,0,'test'), +(1082,2.14647982707,0.128958074579,0.132369756695,2,'test'), +(1083,0.983825690341,0.760567676767,0.472501866211,0,'test'), +(1084,2.29076748513,0.201634075114,0.298552189761,2,'test'), +(1085,1.91463577385,0.175729863213,0.85959636495,1,'test'), +(1086,3.82025184454,0.437118012651,0.618978054447,3,'test'), +(1087,2.47304170663,0.340260802848,0.364391141198,2,'test'), +(1088,1.25455496441,0.967253108561,0.536005462523,0,'test'), +(1089,3.63760498945,0.143026077088,0.703263046348,3,'test'), +(1090,3.84460234516,0.844558532806,0.00661909043169,3,'test'), +(1091,3.45392732798,0.669406139981,0.885732006873,2,'test'), +(1092,2.93495535693,0.10930490808,0.908653095989,2,'test'), +(1093,1.98331749135,0.0882535400293,0.946078195143,1,'test'), +(1094,3.9680856845,0.966462041277,0.0402944564596,3,'test'), +(1095,0.290659179246,0.194297485213,0.310421800189,0,'test'), +(1096,2.09120944355,0.0819000600183,0.0964851466769,2,'test'), +(1097,1.90966647423,0.26938469528,0.800176092462,1,'test'), +(1098,0.667947740593,0.650130518013,0.133481169382,0,'test'), +(1099,0.598799981248,0.546777245345,0.228084931339,0,'test'), +(1100,3.67714598263,0.696724843853,0.99016217802,2,'test'), +(1101,4.39350943143,0.963098077703,0.656057431733,3,'test'), +(1102,3.28503531757,0.277721529731,0.0855206866113,3,'test'), +(1103,1.99887667359,0.233626461848,0.874785809066,1,'test'), +(1104,1.91101832426,0.845672054479,0.255629164566,1,'test'), +(1105,0.291380317685,0.224212470625,0.259167604187,0,'test'), +(1106,1.82320660134,0.688906299559,0.366470055779,1,'test'), +(1107,0.157900239981,0.153494636978,0.0663747165885,0,'test'), +(1108,4.15240458413,0.203598136156,0.974066962779,3,'test'), +(1109,0.383914139369,0.0675988936036,0.56241910153,0,'test'), +(1110,2.17957886796,0.749485695435,0.655814891968,1,'test'), +(1111,3.2171592172,0.180079395418,0.192561215685,3,'test'), +(1112,0.124323413038,0.12426071025,0.00791850918942,0,'test'), +(1113,3.25825751938,0.224807273909,0.182894082671,3,'test'), +(1114,3.36455193881,0.279459553649,0.291705990956,3,'test'), +(1115,3.12351587595,0.0338217310438,0.299489807687,3,'test'), +(1116,0.340572580458,0.254399185041,0.293553053837,0,'test'), +(1117,0.965535481318,0.668854706421,0.544684105603,0,'test'), +(1118,3.76336951688,0.759919080621,0.058740414206,3,'test'), +(1119,3.81255575275,0.226014414267,0.765859868699,3,'test'), +(1120,0.719538347694,0.611849589326,0.328159653779,0,'test'), +(1121,2.14733862851,0.912256084319,0.484853116101,1,'test'), +(1122,2.17903018255,0.54161062828,0.798385592475,1,'test'), +(1123,3.41391817369,0.373945089576,0.199932698972,3,'test'), +(1124,2.07139789396,0.0132006255649,0.241241100133,2,'test'), +(1125,1.27396567499,0.829317694521,0.666819301212,0,'test'), +(1126,1.92785730459,0.816561527714,0.33361021698,1,'test'), +(1127,2.08022965716,0.0797201177562,0.022572979491,2,'test'), +(1128,2.61639878089,0.171675297677,0.666875912903,2,'test'), +(1129,2.97960364108,0.838149032221,0.376104518537,2,'test'), +(1130,1.57755412389,0.564160278862,0.115731780525,1,'test'), +(1131,1.36998934682,0.349782213601,0.142151796385,1,'test'), +(1132,0.774067742417,0.145002962855,0.793136041018,0,'test'), +(1133,1.43169489753,0.098519075917,0.577213843921,1,'test'), +(1134,3.92624777008,0.560181418316,0.605034174049,3,'test'), +(1135,0.854150998656,0.710065890063,0.379585443073,0,'test'), +(1136,2.89559998754,0.950871032626,0.971971684216,1,'test'), +(1137,1.63687681376,0.748051332126,0.942775414207,0,'test'), +(1138,1.19599509042,0.372246012424,0.907606235102,0,'test'), +(1139,3.79314854023,0.758477305598,0.186202133797,3,'test'), +(1140,2.10599171883,0.0710735124624,0.18686413879,2,'test'), +(1141,0.958750475639,0.621845368951,0.58043527347,0,'test'), +(1142,1.35358596509,0.720714030623,0.795532484861,0,'test'), +(1143,0.299947334585,0.0418086191212,0.50807353352,0,'test'), +(1144,2.44076885688,0.395241818264,0.213370660167,2,'test'), +(1145,2.56301090109,0.161338823455,0.633776046908,2,'test'), +(1146,0.644772710713,0.276083084978,0.607198176657,0,'test'), +(1147,1.34836924744,0.226503323313,0.349093002113,1,'test'), +(1148,1.31978983306,0.304683930305,0.122906072896,1,'test'), +(1149,1.25746696237,0.136233381122,0.348186130179,1,'test'), +(1150,0.724149758437,0.694250057599,0.172915299608,0,'test'), +(1151,0.126370676909,0.0528662019835,0.271117087115,0,'test'), +(1152,3.78411746958,0.581438300813,0.450199032397,3,'test'), +(1153,0.231055303897,0.179728128632,0.226555015978,0,'test'), +(1154,1.55875331413,0.506691350559,0.228170908691,1,'test'), +(1155,2.57306109804,0.281518836868,0.539946535474,2,'test'), +(1156,2.45440389177,0.370076877649,0.290391139881,2,'test'), +(1157,4.05697698306,0.504528912975,0.743268504706,3,'test'), +(1158,1.93630539213,0.122331211291,0.902205176686,1,'test'), +(1159,3.52153051984,0.130664199039,0.625193026833,3,'test'), +(1160,4.36008831853,0.844414256488,0.718104492427,3,'test'), +(1161,2.78206832155,0.392198546748,0.624395527536,2,'test'), +(1162,2.36404799147,0.169562478659,0.441005116534,2,'test'), +(1163,1.00210531406,0.801179068421,0.448247973384,0,'test'), +(1164,3.08259763962,0.910128630062,0.415293883359,2,'test'), +(1165,1.91244788438,0.829577477087,0.28787220654,1,'test'), +(1166,1.74761520298,0.714050926034,0.183205559255,1,'test'), +(1167,0.454729661982,0.0993676539884,0.596122477343,0,'test'), +(1168,4.0365271028,0.913301244027,0.351035409574,3,'test'), +(1169,0.620358436199,0.577979346166,0.20586182267,0,'test'), +(1170,2.31095426377,0.900275451751,0.640842267659,1,'test'), +(1171,0.768983511146,0.563570171869,0.453225483923,0,'test'), +(1172,1.42909600208,0.00984544079969,0.647495607154,1,'test'), +(1173,3.15797605105,0.994239362797,0.404643903028,2,'test'), +(1174,1.02791089586,0.132397836382,0.946315517931,0,'test'), +(1175,2.55667921757,0.692636257741,0.929539111512,1,'test'), +(1176,2.34241536018,0.334288252381,0.0901504730846,2,'test'), +(1177,0.204166144697,0.180203757609,0.154797891096,0,'test'), +(1178,2.37895024297,0.140597665172,0.488213659985,2,'test'), +(1179,1.08384168597,0.669247410639,0.643889955919,0,'test'), +(1180,3.34351137358,0.303854272831,0.199140906778,3,'test'), +(1181,2.62720499801,0.613654533898,0.116406460788,2,'test'), +(1182,2.99146169129,0.935599385282,0.236352080601,2,'test'), +(1183,3.24806489965,0.202904892888,0.212508839262,3,'test'), +(1184,1.43885986293,0.119610146156,0.565021872828,1,'test'), +(1185,1.4348463434,0.392190662913,0.206532516774,1,'test'), +(1186,3.70326035736,0.694539226642,0.0933869943682,3,'test'), +(1187,2.7397163059,0.944967033134,0.891487113066,1,'test'), +(1188,1.22574418918,0.163382494405,0.24972323635,1,'test'), +(1189,4.00783992763,0.458713305726,0.741030783371,3,'test'), +(1190,3.34707492133,0.278863551306,0.261173065267,3,'test'), +(1191,0.881470582996,0.204600073996,0.822721404243,0,'test'), +(1192,0.577733373564,0.308336030001,0.519035011886,0,'test'), +(1193,2.80079821395,0.795609570411,0.0720322395761,2,'test'), +(1194,3.64219009838,0.0928460319066,0.741177486486,3,'test'), +(1195,3.06211405139,0.869019178142,0.439425617419,2,'test'), +(1196,0.845842517265,0.833715381087,0.110123277184,0,'test'), +(1197,3.5701116727,0.681793553209,0.942506296791,2,'test'), +(1198,2.71554872782,0.295814257494,0.647869176858,2,'test'), +(1199,0.58950128452,0.166119796657,0.650677714282,0,'test'), +(1200,3.98897638804,0.918941290605,0.264641450712,3,'test'), +(1201,1.4716039368,0.0714251337186,0.632596872488,1,'test'), +(1202,1.47447298872,0.47446807109,0.00221757416388,1,'test'), +(1203,1.67611192053,0.360229419769,0.562034252301,1,'test'), +(1204,3.30759487651,0.159731082506,0.384530615173,3,'test'), +(1205,3.97180824048,0.967915759296,0.062389752216,3,'test'), +(1206,2.38910927864,0.218951478527,0.412501878917,2,'test'), +(1207,2.17012080457,0.711780406383,0.677008418105,1,'test'), +(1208,2.02993557098,0.879075493697,0.38840710252,1,'test'), +(1209,1.98899305597,0.668903793097,0.565764317427,1,'test'), +(1210,0.191586802293,0.143520548369,0.219240174067,0,'test'), +(1211,2.81419332381,0.79288798093,0.14596349844,2,'test'), +(1212,0.703780315308,0.215512697152,0.698761488747,0,'test'), +(1213,3.47054884462,0.955950030688,0.717355430683,2,'test'), +(1214,1.0943515289,0.0828580101206,0.107207829838,1,'test'), +(1215,2.66158930888,0.64103008465,0.143384881455,2,'test'), +(1216,1.53831351249,0.613446245406,0.961700196053,0,'test'), +(1217,3.85699038297,0.480071816039,0.613936940518,3,'test'), +(1218,2.17270037192,0.16519309665,0.0866445340065,2,'test'), +(1219,3.26285277099,0.25198515521,0.104247857426,3,'test'), +(1220,1.88256064237,0.73041733256,0.39005552144,1,'test'), +(1221,3.13328300134,0.958700681403,0.417830491876,2,'test'), +(1222,1.22650604174,0.925802092674,0.548364795614,0,'test'), +(1223,1.00675360567,0.737399898836,0.518992973782,0,'test'), +(1224,1.3615548729,0.233681859821,0.357593362747,1,'test'), +(1225,2.45467810597,0.764502777197,0.830767915109,1,'test'), +(1226,0.393679497682,0.297312219564,0.31043079441,0,'test'), +(1227,3.58904622434,0.281357942249,0.554696567586,3,'test'), +(1228,1.77479434836,0.767401505119,0.0859816447722,1,'test'), +(1229,2.74964077249,0.68578759901,0.252691854787,2,'test'), +(1230,1.99666814403,0.273093960118,0.850631638204,1,'test'), +(1231,4.13284274333,0.163533078832,0.984535253049,3,'test'), +(1232,3.42471704219,0.102916366172,0.567274779994,3,'test'), +(1233,4.13447772653,0.489381560617,0.803178788261,3,'test'), +(1234,2.9247275543,0.833084665274,0.302725765389,2,'test'), +(1235,0.738359633976,0.323256676485,0.644284841892,0,'test'), +(1236,2.6058091408,0.595940065806,0.0993432181666,2,'test'), +(1237,0.320510201713,0.181641380151,0.372651072133,0,'test'), +(1238,3.72081077407,0.321739411837,0.631720952824,3,'test'), +(1239,4.47977446688,0.695313719619,0.885697887126,3,'test'), +(1240,2.25405290657,0.501562771297,0.867461892691,1,'test'), +(1241,1.65875074005,0.870641166862,0.887755356608,0,'test'), +(1242,1.93317879651,0.828160596917,0.324065116282,1,'test'), +(1243,3.56270774508,0.297753740292,0.51473683061,3,'test'), +(1244,1.24104607157,0.0290940964598,0.460382422681,1,'test'), +(1245,3.15506517674,0.643180288949,0.715461311176,2,'test'), +(1246,3.75175346116,0.869188923198,0.939449060868,2,'test'), +(1247,1.90526292672,0.850537456241,0.233934756876,1,'test'), +(1248,2.37027682459,0.0147867353095,0.596229896336,2,'test'), +(1249,1.40269041443,0.151480914026,0.501208041041,1,'test'), +(1250,0.757007724335,0.1823519434,0.758060539096,0,'test'), +(1251,1.28247007315,0.281921722233,0.0234168937502,1,'test'), +(1252,1.60497610596,0.317435213414,0.536228395876,1,'test'), +(1253,2.67195972778,0.0816579363709,0.768310999146,2,'test'), +(1254,0.428524625858,0.366168532859,0.249712020134,0,'test'), +(1255,2.31394364157,0.903973719292,0.640288936559,1,'test'), +(1256,1.70577561334,0.12272157624,0.763579751633,1,'test'), +(1257,2.83897871889,0.782376779319,0.237911621351,2,'test'), +(1258,2.14208843001,0.0568528555986,0.291951321982,2,'test'), +(1259,2.0714639003,0.0632513807487,0.0906229526898,2,'test'), +(1260,2.57995965861,0.428950779716,0.388598608969,2,'test'), +(1261,2.03559304085,0.966559451599,0.26274243899,1,'test'), +(1262,2.27304377019,0.216750927674,0.237261127265,2,'test'), +(1263,1.97879204432,0.948072951627,0.175268630093,1,'test'), +(1264,1.277199669,0.0430766280946,0.483862626066,1,'test'), +(1265,1.41229561405,0.888170989997,0.723964518502,0,'test'), +(1266,0.574801683415,0.565321189889,0.0973678259298,0,'test'), +(1267,4.68160152313,0.822241809901,0.927016565779,3,'test'), +(1268,1.22208292076,0.481814886242,0.860388304501,0,'test'), +(1269,0.529250350219,0.251412080326,0.52710366143,0,'test'), +(1270,3.85163778206,0.844104169881,0.0867963834646,3,'test'), +(1271,3.33412206529,0.34095605773,0.996577145814,2,'test'), +(1272,2.53415835247,0.730357151657,0.896549608674,1,'test'), +(1273,2.06255615285,0.955123741626,0.327768838097,1,'test'), +(1274,2.12921198229,0.914937264844,0.462898171789,1,'test'), +(1275,2.59943560245,0.509604216146,0.299718845432,2,'test'), +(1276,1.27084968419,0.240408956325,0.174472713803,1,'test'), +(1277,3.34053207774,0.791311761713,0.741093999451,2,'test'), +(1278,1.10364889319,0.0455326456546,0.241073116566,1,'test'), +(1279,3.66585060482,0.899968761359,0.875146755384,2,'test'), +(1280,1.1983005446,0.565711657017,0.79535456721,0,'test'), +(1281,1.46069944727,0.0161590361331,0.6667386378,1,'test'), +(1282,2.543283049,0.519909397083,0.152884439756,2,'test'), +(1283,1.67919771186,0.493754211537,0.430631513387,1,'test'), +(1284,0.492359754106,0.212901970472,0.528637667627,0,'test'), +(1285,2.63114297184,0.618117087966,0.114130994369,2,'test'), +(1286,2.06166089723,0.001573158601,0.245128004586,2,'test'), +(1287,1.47550590996,0.2339404212,0.49149312178,1,'test'), +(1288,2.36276007409,0.526961430027,0.91422023827,1,'test'), +(1289,2.10782813881,0.199142919269,0.953249820113,1,'test'), +(1290,3.74400785924,0.243702527993,0.70732265003,3,'test'), +(1291,0.882649103026,0.436403237689,0.66801636607,0,'test'), +(1292,2.03295299136,0.0556422086427,0.988590300741,1,'test'), +(1293,3.45506379811,0.208198380538,0.49685552988,3,'test'), +(1294,3.09014356797,0.0736411275096,0.1284618249,3,'test'), +(1295,4.17939306833,0.798425752628,0.617225498263,3,'test'), +(1296,4.47118556718,0.502491850114,0.984222392078,3,'test'), +(1297,2.46151269575,0.261543140284,0.447179556186,2,'test'), +(1298,1.83094064397,0.796778755943,0.184829348403,1,'test'), +(1299,0.365691676425,0.15507878897,0.458925797331,0,'test'), +(1300,2.02190680521,0.865977719831,0.394878570424,1,'test'), +(1301,1.53553829482,0.46102322759,0.272974480921,1,'test'), +(1302,3.38048029725,0.561745241279,0.904839795752,2,'test'), +(1303,3.19654781993,0.312246265825,0.940373093036,2,'test'), +(1304,2.29463568429,0.243079768063,0.227059279098,2,'test'), +(1305,0.339802343327,0.276684527293,0.251232593495,0,'test'), +(1306,1.5865684912,0.359923450175,0.476072516564,1,'test'), +(1307,0.701085140088,0.569972338199,0.362095017763,0,'test'), +(1308,1.03096397264,0.0307467583864,0.0147381903925,1,'test'), +(1309,0.891796818972,0.385754614549,0.711366434703,0,'test'), +(1310,0.456770772011,0.341207154023,0.33994649283,0,'test'), +(1311,3.63666939478,0.629839703033,0.0826419490526,3,'test'), +(1312,2.20830394584,0.868598773777,0.582842321788,1,'test'), +(1313,0.824860825032,0.772639439844,0.228519988597,0,'test'), +(1314,1.70698780313,0.991370981023,0.845941382194,0,'test'), +(1315,2.17382087932,0.415932460556,0.870567871427,1,'test'), +(1316,4.07187743183,0.214645745578,0.925868071733,3,'test'), +(1317,3.12740137005,0.921886587932,0.453337382215,2,'test'), +(1318,2.338680043,0.814487580638,0.724011368948,1,'test'), +(1319,1.95389357975,0.91389798792,0.199988979263,1,'test'), +(1320,3.21344601883,0.210700325869,0.0523993603574,3,'test'), +(1321,3.01854619866,0.469201578501,0.741177860007,2,'test'), +(1322,3.32791618369,0.323516874982,0.0663272848058,3,'test'), +(1323,1.89279540395,0.451275656791,0.664469523121,1,'test'), +(1324,1.73789215873,0.727425050793,0.102308884959,1,'test'), +(1325,0.990387514373,0.724882441042,0.515271844109,0,'test'), +(1326,4.5587556684,0.637269775392,0.959940567435,3,'test'), +(1327,3.8761078728,0.33920152135,0.732738938128,3,'test'), +(1328,0.281425715506,0.224163025681,0.239296238635,0,'test'), +(1329,0.474572505793,0.467710763062,0.0828356368396,0,'test'), +(1330,1.31027542983,0.748721763941,0.74936884502,0,'test'), +(1331,2.12667327735,0.1834242421,0.971210088111,1,'test'), +(1332,3.6764181282,0.0241589571152,0.807625637956,3,'test'), +(1333,0.11832934962,0.105029491717,0.115325009876,0,'test'), +(1334,2.17413719646,0.128955187511,0.212560600656,2,'test'), +(1335,0.875691486585,0.464505759432,0.641237652632,0,'test'), +(1336,2.35236620121,0.356305333052,0.998028490657,1,'test'), +(1337,4.63970443221,0.919459622908,0.848672380431,3,'test'), +(1338,1.35393602606,0.208138030363,0.381835037287,1,'test'), +(1339,0.647229675545,0.38075324792,0.516213548471,0,'test'), +(1340,3.39366216362,0.0213976734131,0.610134813138,3,'test'), +(1341,3.25008743845,0.0598442103692,0.436168806861,3,'test'), +(1342,4.7515862723,0.837774322496,0.955935117988,3,'test'), +(1343,3.67693301178,0.70216938192,0.987301184976,2,'test'), +(1344,0.275465297145,0.0932686731653,0.426844964805,0,'test'), +(1345,1.68248287677,0.791342675649,0.944002225167,0,'test'), +(1346,3.6148719524,0.396384545945,0.467426364744,3,'test'), +(1347,0.477380071233,0.145294776376,0.576268422575,0,'test'), +(1348,3.98569147707,0.984927272003,0.0276442592572,3,'test'), +(1349,2.60142474933,0.118998491211,0.694569116873,2,'test'), +(1350,2.1721118122,0.1410042582,0.176373336991,2,'test'), +(1351,2.03016208102,0.853919425439,0.419812643431,1,'test'), +(1352,1.74230735132,0.727540444007,0.121519164396,1,'test'), +(1353,2.47135155804,0.0691293250639,0.63420992816,2,'test'), +(1354,2.82020757028,0.62143002917,0.445844750011,2,'test'), +(1355,0.613966487941,0.432989925339,0.425413402001,0,'test'), +(1356,2.40055937688,0.887380945086,0.716364733775,1,'test'), +(1357,2.64700403364,0.618009912313,0.170276602419,2,'test'), +(1358,4.46946835081,0.660810481504,0.899254062714,3,'test'), +(1359,2.95072158373,0.474449990642,0.690124331613,2,'test'), +(1360,0.00815006331396,0.00512158959855,0.0550315701702,0,'test'), +(1361,3.45066018514,0.384338254184,0.257530446658,3,'test'), +(1362,3.30404043722,0.296345506301,0.0877207553364,3,'test'), +(1363,2.92980160315,0.549542598706,0.616651444858,2,'test'), +(1364,2.65290680653,0.480474207679,0.41525004377,2,'test'), +(1365,1.36447623834,0.889405180168,0.68925398669,0,'test'), +(1366,1.15446424543,0.14754400922,0.0831879571144,1,'test'), +(1367,1.11943297648,0.070021245433,0.222287496389,1,'test'), +(1368,1.52291515234,0.479160465729,0.209176209481,1,'test'), +(1369,1.12766723816,0.124767133131,0.0538526232303,1,'test'), +(1370,0.733871102408,0.721084395581,0.113078321647,0,'test'), +(1371,2.65657986158,0.21557092683,0.66408503578,2,'test'), +(1372,1.29795781165,0.242955365473,0.234526003191,1,'test'), +(1373,3.63219710322,0.612258741415,0.141203264152,3,'test'), +(1374,3.26454367003,0.788756869533,0.689773006504,2,'test'), +(1375,2.73699925603,0.681801573434,0.234941870665,2,'test'), +(1376,2.27768333658,0.531895060638,0.863590340347,1,'test'), +(1377,2.38408958671,0.438972120416,0.972171521025,1,'test'), +(1378,0.998419426861,0.989180277753,0.0961204926521,0,'test'), +(1379,1.30049919107,0.622832201038,0.823205314626,0,'test'), +(1380,1.73494754526,0.91869165111,0.903468811943,0,'test'), +(1381,0.329668467737,0.14937217432,0.424613110275,0,'test'), +(1382,3.20272380062,0.79506940588,0.638478186582,2,'test'), +(1383,2.36371059483,0.7371050566,0.791584195285,1,'test'), +(1384,3.11048659347,0.15402470861,0.977988693627,2,'test'), +(1385,4.81931847515,0.924403658299,0.945999374655,3,'test'), +(1386,1.91917529702,0.994131725023,0.961791854819,0,'test'), +(1387,3.22757545871,0.216760922866,0.103992960549,3,'test'), +(1388,0.195293938248,0.195042427678,0.0158590847959,0,'test'), +(1389,3.0039181557,0.0479802256018,0.977720783303,2,'test'), +(1390,4.41574218091,0.489752456341,0.962283598828,3,'test'), +(1391,1.16963911588,0.800229136472,0.607791065591,0,'test'), +(1392,4.11525877853,0.657539649705,0.67654942822,3,'test'), +(1393,3.14322190348,0.131992690114,0.10596798272,3,'test'), +(1394,4.14853599561,0.944543778975,0.451654975216,3,'test'), +(1395,3.49739249244,0.340510121689,0.396083792592,3,'test'), +(1396,1.56825324975,0.478242712724,0.300017561188,1,'test'), +(1397,0.706547642228,0.378243823541,0.572978026356,0,'test'), +(1398,2.39467309839,0.512877944766,0.939039484593,1,'test'), +(1399,2.95932969243,0.589070319108,0.608489419238,2,'test'), +(1400,2.62885622685,0.621595168872,0.0852118417569,2,'test'), +(1401,2.86870853823,0.858388400591,0.10158807821,2,'test'), +(1402,0.964194876582,0.950172677803,0.118415365468,0,'test'), +(1403,4.01100893719,0.805528572939,0.453299420086,3,'test'), +(1404,4.29775508173,0.875049943754,0.650157779294,3,'test'), +(1405,0.661819831854,0.344787756992,0.563056013965,0,'test'), +(1406,2.36697129079,0.318923095973,0.219198984523,2,'test'), +(1407,1.88820855292,0.881654235208,0.0809587407693,1,'test'), +(1408,3.06098619228,0.925783964604,0.367698555438,2,'test'), +(1409,3.7465763021,0.379349377902,0.605992511668,3,'test'), +(1410,0.416554918272,0.241108210561,0.418863590816,0,'test'), +(1411,3.74206510803,0.254901975546,0.697970724661,3,'test'), +(1412,1.91245664193,0.678446469362,0.483745979378,1,'test'), +(1413,4.31216696476,0.682051579131,0.793798076107,3,'test'), +(1414,2.35424340265,0.766143014977,0.766877035561,1,'test'), +(1415,1.83671177296,0.467076150947,0.607976662389,1,'test'), +(1416,1.15074423186,0.671910471959,0.691978149871,0,'test'), +(1417,2.97995417382,0.85992022255,0.346459162486,2,'test'), +(1418,4.08871031675,0.190541755137,0.947717553709,3,'test'), +(1419,1.25676754536,0.0232760898759,0.483209535792,1,'test'), +(1420,0.842997741857,0.837092882606,0.076843082002,0,'test'), +(1421,2.27813484128,0.277712832971,0.0205428407532,2,'test'), +(1422,3.9791479257,0.974427896913,0.0687024656253,3,'test'), +(1423,3.83243454805,0.758217233476,0.272428549483,3,'test'), +(1424,0.632189067936,0.4385639833,0.440028504346,0,'test'), +(1425,2.08441624869,0.0390492147784,0.212995384722,2,'test'), +(1426,3.8379406474,0.780146969764,0.240403156462,3,'test'), +(1427,2.86745889377,0.356949145123,0.714499649158,2,'test'), +(1428,2.95906467116,0.569683886413,0.624003833923,2,'test'), +(1429,3.58005793997,0.992580785253,0.766470583074,2,'test'), +(1430,2.82511252446,0.591256540306,0.48358658393,2,'test'), +(1431,2.57137291,0.524899482178,0.215576964956,2,'test'), +(1432,1.31483575761,0.326459275903,0.994171253709,0,'test'), +(1433,2.65316143843,0.619064951233,0.184652341428,2,'test'), +(1434,2.07594293689,0.994126481547,0.286035758851,1,'test'), +(1435,2.42667854583,0.564059550602,0.928772843718,1,'test'), +(1436,2.61536806245,0.862818143626,0.867496350897,1,'test'), +(1437,3.26986007756,0.867091385877,0.634640600408,2,'test'), +(1438,4.65301034684,0.98471680316,0.817492228515,3,'test'), +(1439,2.32087082668,0.291267228567,0.172056961827,2,'test'), +(1440,0.200554068382,0.182112537713,0.135799597455,0,'test'), +(1441,2.48476719644,0.314766136151,0.412311848344,2,'test'), +(1442,1.04540189843,0.0246166548689,0.144170883187,1,'test'), +(1443,2.79685818111,0.219086959211,0.760112637637,2,'test'), +(1444,3.38523715707,0.584855399465,0.894640574536,2,'test'), +(1445,1.79918318673,0.687326545131,0.334449759449,1,'test'), +(1446,3.9347666664,0.647356851473,0.536106160129,3,'test'), +(1447,3.97831526983,0.855245434679,0.350813105728,3,'test'), +(1448,4.07145051009,0.701107099092,0.608558469659,3,'test'), +(1449,3.78394537246,0.831460355603,0.975953388669,2,'test'), +(1450,2.91066955665,0.0139322679342,0.946962136895,2,'test'), +(1451,1.14287226297,0.115377602742,0.165815138728,1,'test'), +(1452,3.68999913963,0.188577746597,0.708111144547,3,'test'), +(1453,4.72071234267,0.880361840279,0.916706333781,3,'test'), +(1454,2.36225213228,0.293483465932,0.262237804952,2,'test'), +(1455,0.523159257972,0.21654628481,0.553726442535,0,'test'), +(1456,1.48898747774,0.0886172599137,0.632748147231,1,'test'), +(1457,3.48876828423,0.485687105484,0.0555083664135,3,'test'), +(1458,3.50009742398,0.130599104536,0.60786373427,3,'test'), +(1459,2.64676669265,0.646389865677,0.0194120316653,2,'test'), +(1460,2.27171787071,0.270936352166,0.0279556530962,2,'test'), +(1461,4.43449899578,0.630181669692,0.896837402255,3,'test'), +(1462,1.26649493352,0.81287090887,0.673516165098,0,'test'), +(1463,1.74578964612,0.580766913692,0.406229900952,1,'test'), +(1464,3.90220611864,0.0061199497858,0.946618280436,3,'test'), +(1465,3.44537744926,0.201696850468,0.493640151119,3,'test'), +(1466,2.3839438305,0.300648907208,0.288608598782,2,'test'), +(1467,4.51356272689,0.83184911983,0.825659498255,3,'test'), +(1468,1.24909121211,0.209699220295,0.198474159073,1,'test'), +(1469,2.10750879184,0.987161367285,0.346911263233,1,'test'), +(1470,1.150124527,0.509949622305,0.80010930797,0,'test'), +(1471,2.17355636484,0.823064969715,0.59202313732,1,'test'), +(1472,3.50605665288,0.440596072316,0.25585265401,3,'test'), +(1473,3.35620335289,0.979705438418,0.613594258831,2,'test'), +(1474,3.06830411396,0.567505554032,0.707671223046,2,'test'), +(1475,3.76988881871,0.294283995297,0.68964108304,3,'test'), +(1476,0.924975257128,0.858271962097,0.258269810529,0,'test'), +(1477,3.99310749854,0.705350968011,0.53642942735,3,'test'), +(1478,0.909789949935,0.713724764642,0.442792485588,0,'test'), +(1479,0.361108733766,0.241412375267,0.345971615164,0,'test'), +(1480,3.24179447396,0.219700577144,0.148640158816,3,'test'), +(1481,1.3628060111,0.345599448738,0.13117378686,1,'test'), +(1482,0.878603234302,0.57479959972,0.551183848259,0,'test'), +(1483,0.185735390719,0.127154031716,0.242035863053,0,'test'), +(1484,2.18534709441,0.756256656931,0.655049950366,1,'test'), +(1485,2.84401395501,0.432793600259,0.641264652661,2,'test'), +(1486,2.28558545412,0.751462054727,0.730837464414,1,'test'), +(1487,1.17442990989,0.675373923581,0.706438947331,0,'test'), +(1488,4.31189174067,0.998864464079,0.55948840613,3,'test'), +(1489,1.12014578234,0.375917507714,0.862686660746,0,'test'), +(1490,2.52926938436,0.100396949661,0.654883527583,2,'test'), +(1491,1.05905893768,0.872174273476,0.432301589404,0,'test'), +(1492,2.16949267795,0.132034370457,0.193541487786,2,'test'), +(1493,0.446472220334,0.44445027549,0.0449660410119,0,'test'), +(1494,0.665225568085,0.464220582231,0.448335795865,0,'test'), +(1495,0.541144913282,0.242134306969,0.546818622866,0,'test'), +(1496,1.33012777671,0.999276236259,0.575196957964,0,'test'), +(1497,1.56473652153,0.913330207841,0.807097462325,0,'test'), +(1498,4.19676024731,0.657504575947,0.734340296703,3,'test'), +(1499,0.835467872155,0.659799890799,0.419127643273,0,'test'), +(1500,2.70865795836,0.540929153579,0.409547072733,2,'test'), +(1501,3.26260979484,0.254910804421,0.0877438910359,3,'test'), +(1502,0.889868271277,0.218983321615,0.819075667849,0,'test'), +(1503,1.96769735566,0.70266182348,0.514816017793,1,'test'), +(1504,0.566148817332,0.0609481423607,0.710774700571,0,'test'), +(1505,2.52049939635,0.510638166928,0.0993037231085,2,'test'), +(1506,4.55212801831,0.890399077342,0.813467234106,3,'test'), +(1507,1.82921855792,0.799079268281,0.173606709667,1,'test'), +(1508,3.39289383796,0.0172944108764,0.612861670428,3,'test'), +(1509,2.61247204649,0.212388730374,0.632521395778,2,'test'), +(1510,1.06584948888,0.341906551177,0.850848363516,0,'test'), +(1511,3.75085155098,0.0408938283573,0.842589889939,3,'test'), +(1512,0.990922298326,0.571555656449,0.647585239082,0,'test'), +(1513,3.19958843423,0.0614388891608,0.371684738818,3,'test'), +(1514,2.58174874069,0.0144896605169,0.753166037583,2,'test'), +(1515,2.36847257066,0.0566666919384,0.558395808293,2,'test'), +(1516,4.39900164489,0.505284801053,0.945365984073,3,'test'), +(1517,4.43842147192,0.464487877168,0.986880739884,3,'test'), +(1518,3.84621445141,0.305542973964,0.735303663424,3,'test'), +(1519,1.4594292163,0.65288607624,0.898077468854,0,'test'), +(1520,3.08665631068,0.0480890599736,0.196385464599,3,'test'), +(1521,3.39272458114,0.122283508484,0.520039491439,3,'test'), +(1522,4.43841751988,0.931381265562,0.712064782388,3,'test'), +(1523,3.87614204186,0.357459884607,0.720195915884,3,'test'), +(1524,2.93792866883,0.226753327174,0.843312125878,2,'test'), +(1525,1.28144785978,0.218520843148,0.250852579472,1,'test'), +(1526,1.9509532394,0.934140979064,0.129662100613,1,'test'), +(1527,3.08213839947,0.462185660547,0.787370776014,2,'test'), +(1528,1.60092145616,0.550737798236,0.224017092921,1,'test'), +(1529,0.323206292027,0.198345990549,0.353355771819,0,'test'), +(1530,3.76974123886,0.228435789598,0.735734632363,3,'test'), +(1531,3.72388835756,0.60542145578,0.344190211632,3,'test'), +(1532,2.14114024993,0.793264561218,0.589809875057,1,'test'), +(1533,4.29720899005,0.530157120162,0.875814974687,3,'test'), +(1534,2.3439754441,0.00343302276215,0.583560126581,2,'test'), +(1535,3.15912615908,0.48401211986,0.821653235389,2,'test'), +(1536,0.449348934631,0.397332282074,0.228071595245,0,'test'), +(1537,1.75740722659,0.754507890594,0.0538454826154,1,'test'), +(1538,1.63555245169,0.814944429756,0.905874175554,0,'test'), +(1539,0.571064665106,0.461805691162,0.330543452429,0,'test'), +(1540,0.413318305461,0.410329600956,0.0546690452121,0,'test'), +(1541,2.7244603296,0.647161462204,0.278026738627,2,'test'), +(1542,3.97477912165,0.93443400301,0.200860943542,3,'test'), +(1543,1.81179677187,0.327932291415,0.695603680595,1,'test'), +(1544,3.61736266921,0.607356932238,0.100028680739,3,'test'), +(1545,1.79707452445,0.796952682539,0.011038202415,1,'test'), +(1546,1.17043253206,0.812666158682,0.598135748288,0,'test'), +(1547,1.83655843581,0.265980507038,0.755366089237,1,'test'), +(1548,3.23361222466,0.509077566747,0.851196016155,2,'test'), +(1549,0.456097778164,0.357275612767,0.314359929694,0,'test'), +(1550,3.86271799265,0.766083473996,0.310860931376,3,'test'), +(1551,3.80101606604,0.677649232722,0.351236150364,3,'test'), +(1552,3.75890463033,0.910129513378,0.921289920142,2,'test'), +(1553,3.94547523294,0.863614035006,0.286113959699,3,'test'), +(1554,3.03580791913,0.228113202088,0.898718374711,2,'test'), +(1555,4.30371799782,0.585490805266,0.847482856792,3,'test'), +(1556,2.13568278029,0.101578974723,0.184672156978,2,'test'), +(1557,1.94791551676,0.831820467495,0.340727235879,1,'test'), +(1558,0.597812849661,0.31595698549,0.530900992814,0,'test'), +(1559,4.23437046496,0.824150173806,0.640484419132,3,'test'), +(1560,3.99108162181,0.816309927342,0.418057046911,3,'test'), +(1561,3.38422310773,0.37392770809,0.101466248757,3,'test'), +(1562,2.93972996129,0.662088839313,0.526916617669,2,'test'), +(1563,2.76607070171,0.996998416606,0.876967664799,1,'test'), +(1564,0.861818920637,0.54027791888,0.567045855075,0,'test'), +(1565,1.6868238595,0.464422167735,0.471594838566,1,'test'), +(1566,2.90221260755,0.73549405724,0.408311829741,2,'test'), +(1567,4.12434029169,0.738421765487,0.621223410864,3,'test'), +(1568,3.95335309201,0.929171302377,0.155504950501,3,'test'), +(1569,3.41808667635,0.407239817666,0.104148253371,3,'test'), +(1570,1.17902989152,0.663388718053,0.718081592484,0,'test'), +(1571,3.48353565064,0.917877881679,0.752102233052,2,'test'), +(1572,1.45991533119,0.575817436245,0.940264800441,0,'test'), +(1573,3.24275352434,0.242212321869,0.0232637587221,3,'test'), +(1574,1.17225707488,0.0056040898992,0.408231533542,1,'test'), +(1575,2.71119073458,0.707304839499,0.0623369479294,2,'test'), +(1576,3.56813894353,0.880823798299,0.829044718475,2,'test'), +(1577,2.6584577102,0.602501836267,0.236549939609,2,'test'), +(1578,1.03845001944,0.75506541143,0.532338809419,0,'test'), +(1579,2.36007366794,0.894865555576,0.682061663169,1,'test'), +(1580,3.85149734008,0.315429445323,0.732166575826,3,'test'), +(1581,2.24537478405,0.0170172926968,0.477867650453,2,'test'), +(1582,3.22052828059,0.216597281101,0.0626976832806,3,'test'), +(1583,1.87476209063,0.532113907379,0.585361583336,1,'test'), +(1584,3.55223354584,0.487485173777,0.254457014172,3,'test'), +(1585,3.30097105816,0.289539726115,0.106917407571,3,'test'), +(1586,1.5293992334,0.964479486204,0.751611433652,0,'test'), +(1587,4.42000676502,0.949432690618,0.685984019059,3,'test'), +(1588,3.55294388957,0.105832514133,0.668663873289,3,'test'), +(1589,1.66239060901,0.191442907578,0.686256294277,1,'test'), +(1590,1.70077079072,0.727590296825,0.986499109931,0,'test'), +(1591,2.7419894099,0.147804344761,0.770834006216,2,'test'), +(1592,4.24674331258,0.868790505144,0.614778665408,3,'test'), +(1593,4.06825845072,0.992879960521,0.274551434519,3,'test'), +(1594,1.77586448769,0.366385782605,0.639905231328,1,'test'), +(1595,1.23715362614,0.211352754462,0.160626497423,1,'test'), +(1596,1.51580162898,0.503152504265,0.112468327607,1,'test'), +(1597,4.49116862012,0.941377523832,0.741478992476,3,'test'), +(1598,2.7054810231,0.705424492381,0.00751869154257,2,'test'), +(1599,0.270922165976,0.17736872293,0.305865073269,0,'test'), +(1600,2.96633220473,0.933322489613,0.181685759259,2,'test'), +(1601,2.19001488054,0.169482570651,0.143290997239,2,'test'), +(1602,0.747823566306,0.135683525238,0.782393789001,0,'test'), +(1603,3.32489207851,0.594397103572,0.85468998762,2,'test'), +(1604,2.92863754321,0.634107127557,0.542706564957,2,'test'), +(1605,0.536297753681,0.0731327683116,0.680562256792,0,'test'), +(1606,2.67452994544,0.109213737371,0.751875127979,2,'test'), +(1607,3.87350842347,0.210084392625,0.81450845965,3,'test'), +(1608,1.42025407563,0.425636505835,0.9973051538,0,'test'), +(1609,1.54164925613,0.485199693756,0.237591166443,1,'test'), +(1610,2.74902508578,0.748412988859,0.0247405926202,2,'test'), +(1611,2.92141232836,0.888103351227,0.182507471446,2,'test'), +(1612,0.289326942832,0.279335982542,0.0999547912331,0,'test'), +(1613,2.49001494084,0.483143680121,0.082893067963,2,'test'), +(1614,1.22009569255,0.467576902471,0.867478408999,0,'test'), +(1615,0.288866064119,0.022766953088,0.515847953404,0,'test'), +(1616,0.860867631911,0.279112797932,0.762728545407,0,'test'), +(1617,2.08282086073,0.0607585660903,0.148533816475,2,'test'), +(1618,0.713294688192,0.533212213216,0.424361255272,0,'test'), +(1619,2.37411573284,0.372688539977,0.0377782062045,2,'test'), +(1620,1.96993409848,0.818295155511,0.389408452614,1,'test'), +(1621,2.9059984755,0.775787536549,0.36084752867,2,'test'), +(1622,2.96981944661,0.547982422486,0.649489818341,2,'test'), +(1623,2.5638614972,0.830552634531,0.856334550667,1,'test'), +(1624,3.53789475038,0.342783070854,0.441714477384,3,'test'), +(1625,1.85602874213,0.00880749223138,0.920446223252,1,'test'), +(1626,2.93362092688,0.348335217166,0.765039678526,2,'test'), +(1627,1.70901204009,0.658524619586,0.224694059778,1,'test'), +(1628,1.43504247123,0.326918752548,0.328821712603,1,'test'), +(1629,2.45234097623,0.908815253942,0.737241969975,1,'test'), +(1630,3.61420400988,0.394692710512,0.468520329727,3,'test'), +(1631,2.89316951849,0.774306833241,0.344764680982,2,'test'), +(1632,2.23387070772,0.184010109726,0.223294867809,2,'test'), +(1633,0.489833673332,0.238252730504,0.501578451319,0,'test'), +(1634,2.5526313095,0.146261645156,0.637471304719,2,'test'), +(1635,2.73348952494,0.685787240345,0.218408526826,2,'test'), +(1636,1.13954525892,0.745731226639,0.627546039969,0,'test'), +(1637,4.1493937775,0.789465319979,0.599940378301,3,'test'), +(1638,4.81972139041,0.999115351509,0.905873081012,3,'test'), +(1639,3.57665609327,0.492388060899,0.290289566418,3,'test'), +(1640,2.68871742243,0.679060046373,0.0982719494827,2,'test'), +(1641,0.656964324486,0.433009016914,0.473239165297,0,'test'), +(1642,0.963409882427,0.245165866767,0.847492782069,0,'test'), +(1643,3.73861449566,0.659641082575,0.281022086473,3,'test'), +(1644,3.37463206756,0.276540731458,0.313195364106,3,'test'), +(1645,3.72139453411,0.682908974392,0.196177367999,3,'test'), +(1646,2.67581025881,0.921807893633,0.868333095753,1,'test'), +(1647,2.58589280752,0.678071078511,0.952796793135,1,'test'), +(1648,1.13654166705,0.133765349811,0.0526907699719,1,'test'), +(1649,1.51451292679,0.276624911714,0.487737649848,1,'test'), +(1650,2.06600251447,0.906350696482,0.39956453545,1,'test'), +(1651,3.74995995177,0.726547901281,0.153009968584,3,'test'), +(1652,3.29036970971,0.954482936281,0.579557394421,2,'test'), +(1653,2.33825457982,0.67927828752,0.811773547423,1,'test'), +(1654,0.471733645097,0.471276251414,0.0213867642039,0,'test'), +(1655,0.263933955397,0.211596539186,0.228773722728,0,'test'), +(1656,1.91573093667,0.49994860155,0.644811860252,1,'test'), +(1657,3.14291571084,0.127191282221,0.125397083762,3,'test'), +(1658,4.6299665354,0.928187769357,0.837722368117,3,'test'), +(1659,1.1647310818,0.12128712615,0.208432136808,1,'test'), +(1660,2.91255411972,0.319180017713,0.770307796926,2,'test'), +(1661,4.1671723445,0.271813785709,0.946233881657,3,'test'), +(1662,2.35972208311,0.335668872305,0.155090975905,2,'test'), +(1663,3.44886039035,0.190327932596,0.508460871411,3,'test'), +(1664,1.6854013827,0.245746271652,0.663064937279,1,'test'), +(1665,1.11480401495,0.108459909015,0.0796498960008,1,'test'), +(1666,2.91361090391,0.900344100576,0.11518161021,2,'test'), +(1667,1.76099980496,0.148779639813,0.782444991771,1,'test'), +(1668,2.19197753767,0.170030474708,0.148145411552,2,'test'), +(1669,4.04315608788,0.622300513685,0.648733823842,3,'test'), +(1670,2.46888797528,0.398621184455,0.265078838892,2,'test'), +(1671,3.77590936542,0.762511316548,0.115749941144,3,'test'), +(1672,0.820028193775,0.541441511375,0.527813113137,0,'test'), +(1673,2.23996887164,0.105275677621,0.367005713877,2,'test'), +(1674,4.02761138177,0.200410805874,0.909505676672,3,'test'), +(1675,0.229908684095,0.227389653559,0.0501899445654,0,'test'), +(1676,3.21548458331,0.206813875671,0.0931166345981,3,'test'), +(1677,1.14795471295,0.497061376987,0.806779608049,0,'test'), +(1678,3.49978245485,0.29482946887,0.452717335632,3,'test'), +(1679,2.60888430925,0.604075952755,0.0693423138931,2,'test'), +(1680,4.21282674517,0.736723804402,0.690002130985,3,'test'), +(1681,3.30517612917,0.254046950244,0.226117621876,3,'test'), +(1682,3.56976694598,0.561926394697,0.0885468875018,3,'test'), +(1683,3.99272493381,0.89945113472,0.305407595011,3,'test'), +(1684,3.18091640361,0.0608536753063,0.346500690192,3,'test'), +(1685,0.671507882251,0.0303024440781,0.800753044436,0,'test'), +(1686,1.77955605306,0.418383474586,0.600976354337,1,'test'), +(1687,0.41555182918,0.275619917873,0.374074740269,0,'test'), +(1688,3.20700372164,0.24256799556,0.98205688536,2,'test'), +(1689,1.26237424562,0.254421751955,0.0891767552007,1,'test'), +(1690,2.67120147576,0.641951677363,0.171025724371,2,'test'), +(1691,0.178374711104,0.0590424311297,0.345445046244,0,'test'), +(1692,1.18431838703,0.523538497784,0.812883687406,0,'test'), +(1693,4.12197207834,0.949213844037,0.415641954459,3,'test'), +(1694,2.46412849559,0.442853491844,0.145859534298,2,'test'), +(1695,2.59115893553,0.128285443267,0.680348066997,2,'test'), +(1696,4.31460330092,0.943584547559,0.609113087501,3,'test'), +(1697,0.789470507455,0.535311213569,0.50414213659,0,'test'), +(1698,3.78527130955,0.256689739066,0.727036154865,3,'test'), +(1699,0.643137854192,0.203417539813,0.663114103589,0,'test'), +(1700,2.94078908041,0.265690618882,0.821643755853,2,'test'), +(1701,4.23846197683,0.350530670375,0.942301069962,3,'test'), +(1702,3.2038058852,0.0820447493392,0.348942883375,3,'test'), +(1703,0.617590562487,0.615423018897,0.0465568855285,0,'test'), +(1704,0.989064221351,0.762224683677,0.476276744838,0,'test'), +(1705,1.22618882229,0.789801583865,0.66059612353,0,'test'), +(1706,2.43213106016,0.734381937401,0.835313787001,1,'test'), +(1707,1.53393304123,0.0622412994636,0.68679818125,1,'test'), +(1708,0.743045047602,0.546492003664,0.443343031905,0,'test'), +(1709,2.88086462972,0.03770205273,0.918238845284,2,'test'), +(1710,3.77088030912,0.888264879644,0.939476146303,2,'test'), +(1711,2.46785387913,0.486187249976,0.990790910919,1,'test'), +(1712,1.94495089022,0.810321800007,0.366918369963,1,'test'), +(1713,4.03099632821,0.304242742719,0.852498437237,3,'test'), +(1714,2.1144015869,0.688656765499,0.652491242391,1,'test'), +(1715,2.94977856581,0.918701796198,0.176286044869,2,'test'), +(1716,1.34001251393,0.338289139515,0.0415135449808,1,'test'), +(1717,1.68328543143,0.313432340856,0.608155482234,1,'test'), +(1718,2.61732667826,0.593270483022,0.155100597165,2,'test'), +(1719,3.26196964154,0.0108037465141,0.501164538874,3,'test'), +(1720,3.43838347983,0.628091497166,0.900162197974,2,'test'), +(1721,3.33417331914,0.393567421692,0.969848388897,2,'test'), +(1722,4.6673290871,0.994461621143,0.820284990691,3,'test'), +(1723,0.422923721433,0.0401198482079,0.618711462012,0,'test'), +(1724,3.20699240376,0.0318538602635,0.41849557166,3,'test'), +(1725,2.77182196291,0.657159553558,0.338618383061,2,'test'), +(1726,1.19016265701,0.148127473749,0.205024835709,1,'test'), +(1727,0.816059663607,0.68726294123,0.3588826025,0,'test'), +(1728,1.09433988149,0.485579506493,0.780230975415,0,'test'), +(1729,2.64782692464,0.344643075771,0.550621329837,2,'test'), +(1730,0.531982379412,0.473165075717,0.242522790052,0,'test'), +(1731,1.02589339229,0.0725825915045,0.976376362264,0,'test'), +(1732,4.00787431006,0.244717795658,0.873588298002,3,'test'), +(1733,3.23885619004,0.518096790207,0.848975500135,2,'test'), +(1734,3.39541556299,0.748589301914,0.804255097017,2,'test'), +(1735,0.532748665383,0.24935556593,0.532346784956,0,'test'), +(1736,2.36414176029,0.475425182979,0.94271765514,1,'test'), +(1737,2.24073176008,0.807582466299,0.658140785685,1,'test'), +(1738,3.63136785581,0.965939506792,0.81573791687,2,'test'), +(1739,3.6652858561,0.639055277399,0.161958570941,3,'test'), +(1740,2.09827237984,0.588768024152,0.713795738069,1,'test'), +(1741,0.639717357528,0.509223341277,0.361239555214,0,'test'), +(1742,2.50872462901,0.423193451715,0.292457137534,2,'test'), +(1743,1.77981554586,0.710426371711,0.263418249462,1,'test'), +(1744,1.16585948607,0.155153244061,0.103470971803,1,'test'), +(1745,4.30839182541,0.894612223583,0.643257026254,3,'test'), +(1746,2.84079967185,0.837524210816,0.0572316436296,2,'test'), +(1747,4.40199759628,0.416507957968,0.992718307633,3,'test'), +(1748,1.4672330318,0.843609699478,0.78969825397,0,'test'), +(1749,2.73034660114,0.0829004991766,0.804640355663,2,'test'), +(1750,1.71027641877,0.807359823948,0.950219235136,0,'test'), +(1751,3.58888108249,0.689954179036,0.948117557822,2,'test'), +(1752,1.28381182606,0.213726109066,0.264737071432,1,'test'), +(1753,2.01980799022,0.743826873708,0.525339049098,1,'test'), +(1754,2.20142024592,0.341579674998,0.92727588717,1,'test'), +(1755,0.428307176323,0.36591565984,0.249782938735,0,'test'), +(1756,0.140735886642,0.140722739867,0.00362584810863,0,'test'), +(1757,3.0690557193,0.858234088606,0.459153166922,2,'test'), +(1758,1.81350813736,0.738297503772,0.274245571686,1,'test'), +(1759,0.166514145262,0.145989479601,0.143264320961,0,'test'), +(1760,1.57570537806,0.660475356691,0.956676550024,0,'test'), +(1761,3.50126983243,0.460686234233,0.201453712295,3,'test'), +(1762,1.12297886623,0.460412063488,0.813982065371,0,'test'), +(1763,1.93372993449,0.234592353115,0.836144473985,1,'test'), +(1764,1.27051172896,0.632818269548,0.798557110927,0,'test'), +(1765,1.61340027728,0.574646439104,0.196859945583,1,'test'), +(1766,2.11203469392,0.628871592859,0.695099346178,1,'test'), +(1767,1.58825500593,0.127452182723,0.678824589425,1,'test'), +(1768,0.744064434421,0.340818006107,0.635016872464,0,'test'), +(1769,2.80398856699,0.55888460786,0.495079750276,2,'test'), +(1770,0.951480291297,0.592940465471,0.598781951821,0,'test'), +(1771,0.975966657375,0.909664288069,0.257492464562,0,'test'), +(1772,2.98207798086,0.850071427344,0.363327061361,2,'test'), +(1773,1.31283141629,0.358971079624,0.976657737729,0,'test'), +(1774,4.052084222,0.0673736003422,0.992325864652,3,'test'), +(1775,0.950822247591,0.555789747568,0.628516109597,0,'test'), +(1776,2.38373306553,0.381813884674,0.0438084563916,2,'test'), +(1777,3.96631312043,0.761024177408,0.453088228741,3,'test'), +(1778,2.44644177003,0.42868997534,0.133235861114,2,'test'), +(1779,3.22024648946,0.539023698303,0.825362218153,2,'test'), +(1780,0.429475749976,0.380105708629,0.222193702311,0,'test'), +(1781,0.890019023714,0.877201498176,0.113214511164,0,'test'), +(1782,1.51868440783,0.765615671358,0.867795330982,0,'test'), +(1783,2.98909538058,0.90469039619,0.290525359285,2,'test'), +(1784,0.530304314167,0.300221092496,0.479669909073,0,'test'), +(1785,4.43865708545,0.753793824688,0.827564656547,3,'test'), +(1786,2.62652776039,0.0750218387013,0.742634446875,2,'test'), +(1787,3.57241567385,0.264791484852,0.554638791461,3,'test'), +(1788,3.02156413748,0.515237436426,0.7115663715,2,'test'), +(1789,2.00933317658,0.968676531412,0.201634930421,1,'test'), +(1790,1.57805674373,0.872349948731,0.840063566046,0,'test'), +(1791,3.82401844575,0.785991441443,0.195005139193,3,'test'), +(1792,1.25966250061,0.766691656905,0.702118824491,0,'test'), +(1793,4.02384674617,0.0578943846026,0.982828754953,3,'test'), +(1794,1.47026272634,0.469891048644,0.0192789443652,1,'test'), +(1795,3.00656396136,0.251103751529,0.869172140503,2,'test'), +(1796,2.45519130761,0.929647644789,0.724943903222,1,'test'), +(1797,3.10755441621,0.73584357845,0.609680931109,2,'test'), +(1798,2.18944213568,0.527796903638,0.813415780544,1,'test'), +(1799,0.66579588502,0.583030339454,0.287690016453,0,'test'), +(1800,2.12321244523,0.412880077063,0.842812178465,1,'test'), +(1801,0.99495652818,0.795156550994,0.446989907253,0,'test'), +(1802,4.35361280121,0.660027674234,0.832817583254,3,'test'), +(1803,2.57499780129,0.574549227493,0.0211795609136,2,'test'), +(1804,2.15822546556,0.134772660297,0.15314308755,2,'test'), +(1805,4.15816039344,0.248544264213,0.95373797724,3,'test'), +(1806,3.84971033768,0.369875165781,0.692701358381,3,'test'), +(1807,1.45063397298,0.348663873919,0.319327573284,1,'test'), +(1808,2.41817572053,0.103775892312,0.560713677578,2,'test'), +(1809,1.25917054046,0.954816004318,0.551683365837,0,'test'), +(1810,1.68635286017,0.408771331235,0.526860065802,1,'test'), +(1811,3.31125263094,0.610213533449,0.837280775779,2,'test'), +(1812,2.58561142764,0.920951745784,0.815266632368,1,'test'), +(1813,1.5716639192,0.473925355803,0.312631673696,1,'test'), +(1814,1.38321791837,0.315365349329,0.260485256859,1,'test'), +(1815,3.31527325627,0.967490695255,0.58973092255,2,'test'), +(1816,3.31574190404,0.889694370964,0.652723167256,2,'test'), +(1817,2.8280488642,0.78324851792,0.211660922888,2,'test'), +(1818,1.81556805429,0.878734495831,0.967901626438,0,'test'), +(1819,3.56992455607,0.488490231805,0.285366999255,3,'test'), +(1820,1.91418146868,0.870581882259,0.208805139826,1,'test'), +(1821,3.15239689399,0.906835177216,0.495541841597,2,'test'), +(1822,3.53250651291,0.429684492724,0.320658728529,3,'test'), +(1823,2.23934701947,0.20202637587,0.193185516019,2,'test'), +(1824,0.874851176848,0.284333425165,0.768451528519,0,'test'), +(1825,3.06439022494,0.498897517615,0.751992491536,2,'test'), +(1826,4.2582310202,0.955898550518,0.549847678615,3,'test'), +(1827,2.06735179772,0.181027797698,0.941447821187,1,'test'), +(1828,1.99322246037,0.955578660123,0.194020102685,1,'test'), +(1829,1.86725039431,0.0727639735394,0.891339677547,1,'test'), +(1830,2.12440093648,0.949347929034,0.418393364488,1,'test'), +(1831,2.56954018097,0.185832243103,0.619441633948,2,'test'), +(1832,2.967162917,0.404835789642,0.749884742719,2,'test'), +(1833,2.56227642267,0.0682428764691,0.702875199593,2,'test'), +(1834,0.533517180301,0.410112550476,0.351289951216,0,'test'), +(1835,2.01522799775,0.734948802656,0.529414011044,1,'test'), +(1836,1.77347563817,0.350110028397,0.650665512977,1,'test'), +(1837,1.72394694155,0.770235603582,0.976581454855,0,'test'), +(1838,1.52114420823,0.449934413968,0.266851633431,1,'test'), +(1839,1.75892480269,0.727830780903,0.176334970412,1,'test'), +(1840,1.33784471407,0.538841769629,0.893869646227,0,'test'), +(1841,2.59197595391,0.514174596597,0.278928946715,2,'test'), +(1842,4.46812489257,0.919540126441,0.740665083642,3,'test'), +(1843,3.1391116531,0.118966648717,0.141933098269,3,'test'), +(1844,3.07160004706,0.682104273454,0.624095965059,2,'test'), +(1845,1.81300668824,0.624794258533,0.43383456491,1,'test'), +(1846,2.77234205334,0.0336048843039,0.859498207699,2,'test'), +(1847,1.09149294457,0.0913320233632,0.0126854722709,1,'test'), +(1848,0.906410245527,0.678292028422,0.477617228651,0,'test'), +(1849,3.24655684712,0.198551567163,0.219101072475,3,'test'), +(1850,0.512011613356,0.415064782756,0.311362860021,0,'test'), +(1851,3.01505863092,0.920392409647,0.307678763118,2,'test'), +(1852,1.07168445462,0.399877177274,0.819638504068,0,'test'), +(1853,4.33665747697,0.993405931028,0.585876732719,3,'test'), +(1854,3.59656382845,0.14738282132,0.670209674006,3,'test'), +(1855,1.61577189586,0.601241729431,0.120541140006,1,'test'), +(1856,2.87270553719,0.190563050479,0.825919176863,2,'test'), +(1857,3.45690880724,0.386880264207,0.264629066866,3,'test'), +(1858,2.52466612947,0.610379878273,0.956183168227,1,'test'), +(1859,0.282226806598,0.278845746407,0.0581468846181,0,'test'), +(1860,3.9272823452,0.660832493148,0.516187806962,3,'test'), +(1861,1.29118962022,0.117852155891,0.416338161035,1,'test'), +(1862,3.63599865119,0.578217878229,0.240376315307,3,'test'), +(1863,0.643180023509,0.631687089896,0.107205100686,0,'test'), +(1864,3.39830172075,0.335670495646,0.25026231259,3,'test'), +(1865,3.08725331685,0.0829035161016,0.0659530192418,3,'test'), +(1866,3.56649805444,0.510683089389,0.236251910147,3,'test'), +(1867,2.9148248541,0.04878102631,0.93061475799,2,'test'), +(1868,0.772816815381,0.203531384933,0.754510059872,0,'test'), +(1869,3.9943472789,0.0184627979976,0.987868655693,3,'test'), +(1870,3.55205019704,0.583199520132,0.984302126846,2,'test'), +(1871,2.62960873343,0.587550543029,0.205080936224,2,'test'), +(1872,0.296860753433,0.22190350855,0.273783207817,0,'test'), +(1873,3.59301999509,0.396932556958,0.442817612719,3,'test'), +(1874,1.43112899885,0.991501607544,0.663044034215,0,'test'), +(1875,3.58324545314,0.808137348996,0.880402239971,2,'test'), +(1876,4.26226613232,0.40733751791,0.924623498731,3,'test'), +(1877,1.54909356185,0.543466450613,0.0750140735925,1,'test'), +(1878,0.481378829381,0.377889588782,0.32169743642,0,'test'), +(1879,2.55777137191,0.531804823133,0.161141393733,2,'test'), +(1880,0.830503856349,0.722299506142,0.328944296511,0,'test'), +(1881,4.13030895958,0.915665328446,0.463296482972,3,'test'), +(1882,2.09997484994,0.171653243927,0.963494476379,1,'test'), +(1883,0.580041502404,0.375908635537,0.451810653777,0,'test'), +(1884,3.77965047085,0.779558763098,0.0095764163488,3,'test'), +(1885,1.01450272452,0.839165050242,0.418733416725,0,'test'), +(1886,0.793008165755,0.622390143409,0.413059344823,0,'test'), +(1887,1.60178197497,0.508089302973,0.3060925873,1,'test'), +(1888,1.24841338579,0.0778431423252,0.413001505397,1,'test'), +(1889,2.40331373503,0.313684167518,0.299381975929,2,'test'), +(1890,1.50430715803,0.524233358709,0.989986767243,0,'test'), +(1891,3.63386269817,0.511300916934,0.35008824778,3,'test'), +(1892,1.70309712414,0.383261258423,0.565540330763,1,'test'), +(1893,3.31470762181,0.0615360151699,0.503161610861,3,'test'), +(1894,0.90762802197,0.778166806954,0.359807191445,0,'test'), +(1895,3.12897336919,0.128115412922,0.0292908904713,3,'test'), +(1896,0.316575566382,0.264321449497,0.228591594081,0,'test'), +(1897,3.80856892667,0.78734912918,0.145670166773,3,'test'), +(1898,0.202416441487,0.108914952777,0.30578013132,0,'test'), +(1899,3.0519453895,0.412263384389,0.799801228503,2,'test'), +(1900,1.72167749455,0.80401165088,0.957948768813,0,'test'), +(1901,3.18616212117,0.970546137106,0.464344682386,2,'test'), +(1902,1.06480111828,0.701032111689,0.603132660853,0,'test'), +(1903,1.22087548687,0.013450720439,0.455439091904,1,'test'), +(1904,1.56110722627,0.154926334418,0.637323224003,1,'test'), +(1905,0.845281388391,0.842336799226,0.0542640688227,0,'test'), +(1906,1.45770584801,0.294279931398,0.404259714305,1,'test'), +(1907,2.54473099894,0.925446300536,0.786946439349,1,'test'), +(1908,0.920019959942,0.919192698564,0.0287621518304,0,'test'), +(1909,2.59058206881,0.839900558356,0.866418784686,1,'test'), +(1910,3.49982162654,0.246260551634,0.50354848317,3,'test'), +(1911,3.72397200939,0.528035121419,0.442647588911,3,'test'), +(1912,3.27348204603,0.999684288645,0.523256875147,2,'test'), +(1913,2.17554874814,0.302843942269,0.934186708252,1,'test'), +(1914,2.30428222973,0.991373397452,0.559382545563,1,'test'), +(1915,3.81636769532,0.0255093043848,0.88930219326,3,'test'), +(1916,2.58202267207,0.257164860612,0.569962991305,2,'test'), +(1917,1.45978945987,0.133290832492,0.571400583981,1,'test'), +(1918,1.90525769744,0.132046833686,0.879324094831,1,'test'), +(1919,3.57785224284,0.572946082307,0.0700439899496,3,'test'), +(1920,4.48792216584,0.824553555107,0.814474438358,3,'test'), +(1921,0.642658807062,0.331427273232,0.557881290087,0,'test'), +(1922,1.93596222633,0.150516874745,0.886253548137,1,'test'), +(1923,0.836681823737,0.74897599821,0.296151693438,0,'test'), +(1924,0.0997209269147,0.0128509258339,0.294737172886,0,'test'), +(1925,3.00108577109,0.487481814063,0.716661675431,2,'test'), +(1926,0.177437614564,0.0769605187886,0.316981223065,0,'test'), +(1927,2.37683859581,0.0911150504603,0.534531145349,2,'test'), +(1928,0.242662108023,0.140789354709,0.31917511387,0,'test'), +(1929,2.94549723169,0.82287091288,0.350180408943,2,'test'), +(1930,0.9572773328,0.895988371518,0.247566074577,0,'test'), +(1931,4.49091232242,0.959209411196,0.729179615201,3,'test'), +(1932,0.315089572492,0.161806799396,0.391513439228,0,'test'), +(1933,3.49132905665,0.0198419518134,0.686649186149,3,'test'), +(1934,3.00112265453,0.951181525607,0.223475119237,2,'test'), +(1935,1.28571235608,0.147935028469,0.37118368446,1,'test'), +(1936,3.25703774402,0.553406557909,0.838827268339,2,'test'), +(1937,0.109872010282,0.102110906041,0.088097129582,0,'test'), +(1938,1.60261009975,0.601178781342,0.037832768992,1,'test'), +(1939,1.61253128986,0.27035172698,0.584961163564,1,'test'), +(1940,2.9931489255,0.0876261225681,0.951589618972,2,'test'), +(1941,2.31519171933,0.185954788371,0.359495383786,2,'test'), +(1942,3.71870129659,0.71470669129,0.0632028899412,3,'test'), +(1943,0.966130879678,0.738540194092,0.477064655561,0,'test'), +(1944,2.75531025075,0.471953434885,0.532312704961,2,'test'), +(1945,1.21280887784,0.18855272932,0.155743855478,1,'test'), +(1946,1.61281191429,0.612012358779,0.0282764126251,1,'test'), +(1947,4.66311561586,0.892511485737,0.877840606329,3,'test'), +(1948,2.06761163408,0.722214718191,0.587704786343,1,'test'), +(1949,4.31885566533,0.92842568248,0.624843966806,3,'test'), +(1950,0.991633719754,0.987935199597,0.0608154598515,0,'test'), +(1951,3.53047693046,0.135939888,0.628121837272,3,'test'), +(1952,0.808258374729,0.437176558169,0.609164851711,0,'test'), +(1953,4.68418768294,0.915593636552,0.876694956292,3,'test'), +(1954,0.604170759826,0.55145382796,0.229601680888,0,'test'), +(1955,2.26908090741,0.246761227421,0.149397724181,2,'test'), +(1956,0.580859904882,0.542238598322,0.196523043329,0,'test'), +(1957,1.72547859391,0.621955518682,0.321750019778,1,'test'), +(1958,2.83773054571,0.83504584089,0.0518141372296,2,'test'), +(1959,2.05442006097,0.914189764609,0.374473358677,1,'test'), +(1960,2.57345707287,0.655667638752,0.958013274502,1,'test'), +(1961,4.1230147686,0.551619483355,0.755906928957,3,'test'), +(1962,0.222690643459,0.219726692105,0.0544421835888,0,'test'), +(1963,0.744276778807,0.288878954802,0.674831700505,0,'test'), +(1964,2.16698560453,0.869970054018,0.544991330675,1,'test'), +(1965,3.2854290557,0.238157763797,0.217419621717,3,'test'), +(1966,3.75542631771,0.580238334609,0.418554635745,3,'test'), +(1967,2.01549948959,0.6292398158,0.621497927423,1,'test'), +(1968,2.79652846435,0.780581573159,0.126281000926,2,'test'), +(1969,1.96778109881,0.501755816748,0.68266044419,1,'test'), +(1970,0.0493477033576,0.04925887161,0.00942505955309,0,'test'), +(1971,1.52012270872,0.0851256599412,0.659543060593,1,'test'), +(1972,2.16470723813,0.828280082698,0.58002340938,1,'test'), +(1973,3.08855181657,0.829953378098,0.508525750061,2,'test'), +(1974,2.21861872737,0.0853745580489,0.365026258406,2,'test'), +(1975,2.09043802902,0.451247940358,0.799493645168,1,'test'), +(1976,3.10208578256,0.101779209716,0.0175092216619,3,'test'), +(1977,1.9140333904,0.432805735554,0.693705740819,1,'test'), +(1978,1.51344393524,0.444334238151,0.262887232644,1,'test'), +(1979,0.378488692208,0.353390532075,0.158423988499,0,'test'), +(1980,0.530527156737,0.454401343061,0.275909067766,0,'test'), +(1981,3.58540324832,0.799741074321,0.886375864971,2,'test'), +(1982,1.38864960164,0.020642043746,0.606636264899,1,'test'), +(1983,1.86705427687,0.846018409714,0.145037468123,1,'test'), +(1984,0.75531736757,0.54544203487,0.458121526125,0,'test'), +(1985,3.32803839577,0.19236356018,0.368340651562,3,'test'), +(1986,2.63135685292,0.497690204239,0.365604497626,2,'test'), +(1987,1.61306902879,0.612978207831,0.0095300028424,1,'test'), +(1988,2.69688358862,0.521792214845,0.41843921157,2,'test'), +(1989,1.19374842338,0.588523622104,0.777961953617,0,'test'), +(1990,1.55052182524,0.535838324388,0.121175496079,1,'test'), +(1991,0.521803046661,0.324592897743,0.444083493183,0,'test'), +(1992,1.61824698903,0.534816460534,0.288843432488,1,'test'), +(1993,2.75195464745,0.630966918201,0.347832904212,2,'test'), +(1994,0.642550668278,0.640947783993,0.040036037329,0,'test'), +(1995,0.199940949396,0.18066076746,0.138853094803,0,'test'), +(1996,2.28505531639,0.186840748362,0.313392035682,2,'test'), +(1997,1.21174260286,0.0752060834306,0.369508483575,1,'test'), +(1998,2.37858881946,0.372735963003,0.0765039636542,2,'test'), +(1999,1.6766399934,0.622827029704,0.231976213636,1,'test'); + diff --git a/src/pg/test/sql/06_segmentation_test.sql b/src/pg/test/sql/06_segmentation_test.sql new file mode 100644 index 0000000..fb6bd2b --- /dev/null +++ b/src/pg/test/sql/06_segmentation_test.sql @@ -0,0 +1,5 @@ +\pset format unaligned +\set ECHO all +\i test/fixtures/ml_values.sql +SELECT cdb_crankshaft._cdb_random_seeds(1234); +SELECT prediction from cdb_crankshaft.CDB_CreateAndPredictSegment('select target,x1,x2,x3 from ml_values where class= $$train$$','target','select cartodb_id, target,x1,x2,x3 from ml_values where class=$$test$$') limit 20 From faa899cf8707adab4fc49834afa0a4d566abb481 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 23 Jun 2016 10:11:59 +0200 Subject: [PATCH 106/183] Fix installation for development mode --- src/py/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/Makefile b/src/py/Makefile index b584645..cc3c67e 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -2,7 +2,7 @@ include ../../Makefile.global # Install the package locally for development install: - pip install ./crankshaft + pip install --upgrade ./crankshaft # Test develpment install test: From 03aada87583302e5be27e165ece4192199414c33 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 23 Jun 2016 20:23:04 +0000 Subject: [PATCH 107/183] Adding a segmentation version that doesn't use query string passing --- src/pg/sql/05_segmentation.sql | 39 ++++++++++++++++++- .../crankshaft/segmentation/segmentation.py | 16 ++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 3979ca4..67b27ec 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -1,3 +1,41 @@ + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1 + ) +RETURNS TABLE( cartodb_id Numeric, prediction Numeric , accuracy Numeric) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = { 'n_estimators' : n_estimators, + 'max_depth' : max_depth, + 'subsample' : subsample, + 'learning_rate' : learning_rate, + 'min_samples_leaf' : min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg( np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ Language plpythonu; + CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( query TEXT, @@ -16,4 +54,3 @@ AS $$ model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} return create_and_predict_segment(query,variable_name,target_table, model_params) $$ LANGUAGE plpythonu; - diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 9ba4c7c..4dbcd69 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -29,6 +29,18 @@ def get_data(variable, feature_columns, query): return replace_nan_with_mean(target), replace_nan_with_mean(features) + +def create_and_predict_segment_agg(target, features, target_features, target_ids,model_parameters): + clean_target = replace_nan_with_mean(target) + clean_features = replace_nan_with_mean(features) + target_features = replace_nan_with_mean(target_features) + + model, accuracy = train_model(clean_target,clean_features, model_parameters, 0.2) + prediction = model.predict(target_features) + return zip(target_ids, prediction, np.full(prediction.shape, accuracy)) + + + def create_and_predict_segment(query,variable,target_query,model_params): """ generate a segment with machine learning @@ -48,9 +60,7 @@ def create_and_predict_segment(query,variable,target_query,model_params): def train_model(target,features,model_params,test_split): features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) model = GradientBoostingRegressor(**model_params) - plpy.notice('training the model: fitting to data') model.fit(features_train, target_train) - plpy.notice('model trained') accuracy = calculate_model_accuracy(model,features,target) return model, accuracy @@ -82,10 +92,8 @@ def predict_segment(model,features,target_query): #Need to fix this. Should be global mean. This will cause weird effects batch = replace_nan_with_mean(batch) - plpy.notice(len(batch)) prediction = model.predict(batch) results.append(prediction) - plpy.notice('predicting: predicted') cartodb_ids = plpy.execute('select array_agg(cartodb_id order by cartodb_id) as cartodb_ids from ({0}) a '.format(target_query))[0]['cartodb_ids'] From 8cbc29a3e671317b19fccadd540e6062c0c16b61 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 23 Jun 2016 20:23:39 +0000 Subject: [PATCH 108/183] adding helper method to be abel to pass 2D arrays from postgresql to python --- src/pg/sql/04_py_agg.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/pg/sql/04_py_agg.sql diff --git a/src/pg/sql/04_py_agg.sql b/src/pg/sql/04_py_agg.sql new file mode 100644 index 0000000..c38e323 --- /dev/null +++ b/src/pg/sql/04_py_agg.sql @@ -0,0 +1,19 @@ +CREATE OR REPLACE FUNCTION + CDB_PyAggS(current_state Numeric[], current_row Numeric[]) + returns NUMERIC[] as $$ + BEGIN + if array_upper(current_state,1) is null then + RAISE NOTICE 'setting state %',array_upper(current_row,1); + current_state[1] = array_upper(current_row,1); + end if; + return array_cat(current_state,current_row) ; + END + $$ LANGUAGE plpgsql; + + +CREATE AGGREGATE CDB_PyAgg(NUMERIC[])( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" +); + From 15fc0bb6834e046e979698c7c3e1a28c200c0a73 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 23 Jun 2016 21:17:51 +0000 Subject: [PATCH 109/183] segmentation documentation --- doc/12_segmentation.md | 84 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 doc/12_segmentation.md diff --git a/doc/12_segmentation.md b/doc/12_segmentation.md new file mode 100644 index 0000000..e57ba83 --- /dev/null +++ b/doc/12_segmentation.md @@ -0,0 +1,84 @@ + +## Segmentation Functions + +### CDB_CreateAndPredictSegment (query TEXT,variable_name TEXT,target_query TEXT) + +This function trains a [Gradient Boosting](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) model to attempt to predict the target data and then generates predictions for new data. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| query | TEXT | The input query to train the algorithum, should have both the variable of interest and the features that will be used to predict it| +| variablei\_name| TEXT | Specify the variable in the query to predict, all other columns are assumed to be features | +| target\_table | TEXT | The query which returns the cartodb\_id and features for the rows your would like to predict the target variable for | +| n\_estimators(optional) | INTEGER DEFAULT 1200| Number of estimators to be used | +| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | +| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample paramter for GradientBooster| +| learning\_rate(optional) | DOUBLE PRECISION DEFAULT 0.01| Learning rate for the GradientBooster| +| min\_samples\_leaf(optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf| + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query| +| prediction | NUMERIC | The predicted value of the variable of interest | +| accuracy | NUMERIC | The mean squared accuracy of the model. | + +#### Example Usage + +```sql +SELECT * from cdb_crankshaft.CDB_CreateAndPredictSegment( +'SELECT agg, median_rent::numeric, male_pop::numeric, female_pop::numeric from late_night_agg', +'agg', +'select ROW_NUMBER ( ) over () as cartodb_id, median_rent, male_pop, female_pop from ml_learning_ny'); +``` + + +### CDB_CreateAndPredictSegment (target NUMERIC[],train_features NUMERIC[], prediction_features Numeric[], prediction_ids NUMERIC[]) + +This function trains a [Gradient Boosting](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) model to attempt to predict the target data and then generates predictions for new data. + + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| target | NUMERIC[] | An array of target values of the variable you want to predict| +| train\_features| NUMERIC[] | 1D array of length nfeatures\* n\_rows + 1 with the first entry in the array being the number of features in each row. These are the features the model will be trained on. CDB\_Crankshaft.CDB_pyAgg(Array[freature1, feature2, feature3]::Numeric[]) can be used to construct this. | +| prediction\_features | NUMERIC[]] | 1D array of length nfeatures\* n\_rows\_ + 1 with the first entry in the array being the number of features in each row. These are the features that will be used to predict the target variable CDB\_Crankshaft.CDB\_pyAgg(Array[freature1, feature2, feature3]::Numeric[]) can be used to construct this. | + +| prediction\_ids | NUMERIC[]] | 1D array of length n\_rows with the ids that can use used to rejoin the data with inputs | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query| +| prediction | NUMERIC | The predicted value of the variable of interest | +| accuracy | NUMERIC | The mean squared accuracy of the model. | +| n\_estimators(optional) | INTEGER DEFAULT 1200| Number of estimators to be used | +| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | +| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample paramter for GradientBooster| +| learning\_rate(optional) | DOUBLE PRECISION DEFAULT 0.01| Learning rate for the GradientBooster| +| min\_samples\_leaf(optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf| + +#### Example Usage + +```sql +WITH training AS ( + SELECT array_agg(agg) AS target, + cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) AS features + FROM late_night_agg), +target AS ( + SELECT cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) AS features, + array_agg(cartodb_id ) AS cartodb_ids FROM late_night_agg) + +SELECT cdb_crankshaft.CDB_CreateAndPredictSegment2(training.target, training.features, target.features, targetcartodb_ids) +FROM training, target; +````` From 642935e44a4fff8beec13139793515bcc005f047 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 23 Jun 2016 21:33:17 +0000 Subject: [PATCH 110/183] pyAgg documentation --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index abd4dae..de6aeee 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + install_requires=['joblib==0.8.3', 'numpy==1.11.0', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], requires=['pysal', 'numpy', 'sklearn'], From de7d56dc290ed2731a78faa5ebcf506fb449be10 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 24 Jun 2016 17:00:08 -0400 Subject: [PATCH 111/183] doc edits --- doc/12_segmentation.md | 64 ++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/doc/12_segmentation.md b/doc/12_segmentation.md index e57ba83..b21c850 100644 --- a/doc/12_segmentation.md +++ b/doc/12_segmentation.md @@ -1,7 +1,7 @@ ## Segmentation Functions -### CDB_CreateAndPredictSegment (query TEXT,variable_name TEXT,target_query TEXT) +### CDB_CreateAndPredictSegment (query TEXT, variable_name TEXT, target_query TEXT) This function trains a [Gradient Boosting](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) model to attempt to predict the target data and then generates predictions for new data. @@ -9,14 +9,14 @@ This function trains a [Gradient Boosting](http://scikit-learn.org/stable/module | Name | Type | Description | |------|------|-------------| -| query | TEXT | The input query to train the algorithum, should have both the variable of interest and the features that will be used to predict it| -| variablei\_name| TEXT | Specify the variable in the query to predict, all other columns are assumed to be features | -| target\_table | TEXT | The query which returns the cartodb\_id and features for the rows your would like to predict the target variable for | -| n\_estimators(optional) | INTEGER DEFAULT 1200| Number of estimators to be used | -| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | -| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample paramter for GradientBooster| -| learning\_rate(optional) | DOUBLE PRECISION DEFAULT 0.01| Learning rate for the GradientBooster| -| min\_samples\_leaf(optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf| +| query | TEXT | The input query to train the algorithm, which should have both the variable of interest and the features that will be used to predict it | +| variable\_name| TEXT | Specify the variable in the query to predict, all other columns are assumed to be features | +| target\_table | TEXT | The query which returns the `cartodb_id` and features for the rows your would like to predict the target variable for | +| n\_estimators (optional) | INTEGER DEFAULT 1200 | Number of estimators to be used. Values should be between 1 and x. | +| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth. Values should be between 1 and n. | +| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample parameter for GradientBooster. Values should be within the range 0 to 1. | +| learning\_rate (optional) | DOUBLE PRECISION DEFAULT 0.01 | Learning rate for the GradientBooster. Values should be between 0 and 1 (??) | +| min\_samples\_leaf (optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf. Values should range from x to y | #### Returns @@ -24,21 +24,20 @@ A table with the following columns. | Column Name | Type | Description | |-------------|------|-------------| -| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query| +| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query | | prediction | NUMERIC | The predicted value of the variable of interest | | accuracy | NUMERIC | The mean squared accuracy of the model. | #### Example Usage ```sql -SELECT * from cdb_crankshaft.CDB_CreateAndPredictSegment( -'SELECT agg, median_rent::numeric, male_pop::numeric, female_pop::numeric from late_night_agg', +SELECT * from cdb_crankshaft.CDB_CreateAndPredictSegment( +'SELECT agg, median_rent::numeric, male_pop::numeric, female_pop::numeric FROM late_night_agg', 'agg', -'select ROW_NUMBER ( ) over () as cartodb_id, median_rent, male_pop, female_pop from ml_learning_ny'); +'SELECT row_number() OVER () As cartodb_id, median_rent, male_pop, female_pop FROM ml_learning_ny'); ``` - -### CDB_CreateAndPredictSegment (target NUMERIC[],train_features NUMERIC[], prediction_features Numeric[], prediction_ids NUMERIC[]) +### CDB_CreateAndPredictSegment(target numeric[], train_features numeric[], prediction_features numeric[], prediction_ids numeric[]) This function trains a [Gradient Boosting](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) model to attempt to predict the target data and then generates predictions for new data. @@ -47,11 +46,10 @@ This function trains a [Gradient Boosting](http://scikit-learn.org/stable/module | Name | Type | Description | |------|------|-------------| -| target | NUMERIC[] | An array of target values of the variable you want to predict| -| train\_features| NUMERIC[] | 1D array of length nfeatures\* n\_rows + 1 with the first entry in the array being the number of features in each row. These are the features the model will be trained on. CDB\_Crankshaft.CDB_pyAgg(Array[freature1, feature2, feature3]::Numeric[]) can be used to construct this. | -| prediction\_features | NUMERIC[]] | 1D array of length nfeatures\* n\_rows\_ + 1 with the first entry in the array being the number of features in each row. These are the features that will be used to predict the target variable CDB\_Crankshaft.CDB\_pyAgg(Array[freature1, feature2, feature3]::Numeric[]) can be used to construct this. | - -| prediction\_ids | NUMERIC[]] | 1D array of length n\_rows with the ids that can use used to rejoin the data with inputs | +| target | numeric[] | An array of target values of the variable you want to predict| +| train\_features| numeric[] | 1D array of length n features \* n\_rows + 1 with the first entry in the array being the number of features in each row. These are the features the model will be trained on. CDB\_Crankshaft.CDB_pyAgg(Array[feature1, feature2, feature3]::numeric[]) can be used to construct this. | +| prediction\_features | numeric[] | 1D array of length nfeatures\* n\_rows\_ + 1 with the first entry in the array being the number of features in each row. These are the features that will be used to predict the target variable CDB\_Crankshaft.CDB\_pyAgg(Array[feature1, feature2, feature3]::numeric[]) can be used to construct this. | +| prediction\_ids | numeric[] | 1D array of length n\_rows with the ids that can use used to re-join the data with inputs | #### Returns @@ -62,23 +60,23 @@ A table with the following columns. | cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query| | prediction | NUMERIC | The predicted value of the variable of interest | | accuracy | NUMERIC | The mean squared accuracy of the model. | -| n\_estimators(optional) | INTEGER DEFAULT 1200| Number of estimators to be used | +| n\_estimators (optional) | INTEGER DEFAULT 1200 | Number of estimators to be used | | max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | -| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample paramter for GradientBooster| -| learning\_rate(optional) | DOUBLE PRECISION DEFAULT 0.01| Learning rate for the GradientBooster| -| min\_samples\_leaf(optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf| +| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample parameter for GradientBooster| +| learning\_rate (optional) | DOUBLE PRECISION DEFAULT 0.01 | Learning rate for the GradientBooster | +| min\_samples\_leaf (optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf | #### Example Usage ```sql -WITH training AS ( - SELECT array_agg(agg) AS target, - cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) AS features - FROM late_night_agg), +WITH training As ( + SELECT array_agg(agg) As target, + cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) As features + FROM late_night_agg), target AS ( - SELECT cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) AS features, - array_agg(cartodb_id ) AS cartodb_ids FROM late_night_agg) + SELECT cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) As features, + array_agg(cartodb_id) As cartodb_ids FROM late_night_agg) -SELECT cdb_crankshaft.CDB_CreateAndPredictSegment2(training.target, training.features, target.features, targetcartodb_ids) -FROM training, target; -````` +SELECT cdb_crankshaft.CDB_CreateAndPredictSegment2(training.target, training.features, target.features, target.cartodb_ids) +FROM training, target; +``` From a177bf5620f29854b5d70d48bc48a944da027c02 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 08:54:28 -0400 Subject: [PATCH 112/183] minor doc updates --- doc/12_segmentation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/12_segmentation.md b/doc/12_segmentation.md index b21c850..2404730 100644 --- a/doc/12_segmentation.md +++ b/doc/12_segmentation.md @@ -1,7 +1,7 @@ ## Segmentation Functions -### CDB_CreateAndPredictSegment (query TEXT, variable_name TEXT, target_query TEXT) +### CDB_CreateAndPredictSegment(query TEXT, variable_name TEXT, target_query TEXT) This function trains a [Gradient Boosting](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) model to attempt to predict the target data and then generates predictions for new data. @@ -57,7 +57,7 @@ A table with the following columns. | Column Name | Type | Description | |-------------|------|-------------| -| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query| +| cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query | | prediction | NUMERIC | The predicted value of the variable of interest | | accuracy | NUMERIC | The mean squared accuracy of the model. | | n\_estimators (optional) | INTEGER DEFAULT 1200 | Number of estimators to be used | @@ -77,6 +77,6 @@ target AS ( SELECT cdb_crankshaft.CDB_PyAgg(Array[median_rent, male_pop, female_pop]::Numeric[]) As features, array_agg(cartodb_id) As cartodb_ids FROM late_night_agg) -SELECT cdb_crankshaft.CDB_CreateAndPredictSegment2(training.target, training.features, target.features, target.cartodb_ids) +SELECT cdb_crankshaft.CDB_CreateAndPredictSegment(training.target, training.features, target.features, target.cartodb_ids) FROM training, target; ``` From dae406927f1e1894dfe00c4a60aa42870a1fdb1b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 09:56:12 -0400 Subject: [PATCH 113/183] formatting updates --- src/pg/sql/05_segmentation.sql | 37 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 67b27ec..7dac003 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -9,32 +9,31 @@ CREATE OR REPLACE FUNCTION max_depth INTEGER DEFAULT 3, subsample DOUBLE PRECISION DEFAULT 0.5, learning_rate DOUBLE PRECISION DEFAULT 0.01, - min_samples_leaf INTEGER DEFAULT 1 - ) -RETURNS TABLE( cartodb_id Numeric, prediction Numeric , accuracy Numeric) + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC) AS $$ - import numpy as np - import plpy + import numpy as np + import plpy from crankshaft.segmentation import create_and_predict_segment_agg - model_params = { 'n_estimators' : n_estimators, - 'max_depth' : max_depth, - 'subsample' : subsample, - 'learning_rate' : learning_rate, - 'min_samples_leaf' : min_samples_leaf} - + model_params = {'n_estimators': n_estimators, + 'max_depth': max_depth, + 'subsample': subsample, + 'learning_rate': learning_rate, + 'min_samples_leaf': min_samples_leaf} + def unpack2D(data): dimension = data.pop(0) a = np.array(data, dtype=float) return a.reshape(len(a)/dimension, dimension) - return create_and_predict_segment_agg( np.array(target, dtype=float), - unpack2D(features), - unpack2D(target_features), + return create_and_predict_segment_agg(np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), target_ids, model_params) -$$ Language plpythonu; +$$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment ( @@ -45,12 +44,10 @@ CREATE OR REPLACE FUNCTION max_depth INTEGER DEFAULT 3, subsample DOUBLE PRECISION DEFAULT 0.5, learning_rate DOUBLE PRECISION DEFAULT 0.01, - min_samples_leaf INTEGER DEFAULT 1 - - ) -RETURNS TABLE (cartodb_id text, prediction Numeric,accuracy Numeric ) + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC) AS $$ from crankshaft.segmentation import create_and_predict_segment - model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} return create_and_predict_segment(query,variable_name,target_table, model_params) $$ LANGUAGE plpythonu; From c80975fe46ca7ab2ee7a5a161135ce95bf68b3d1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 09:56:58 -0400 Subject: [PATCH 114/183] updating formatting --- .../crankshaft/segmentation/segmentation.py | 103 ++++++++++++++---- 1 file changed, 79 insertions(+), 24 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 4dbcd69..da81842 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -9,28 +9,65 @@ from sklearn.ensemble import GradientBoostingRegressor from sklearn import metrics from sklearn.cross_validation import train_test_split -# High level interface --------------------------------------- +# Lower level functions +#---------------------- def replace_nan_with_mean(array): - indices = np.where(np.isnan(array)) #returns an array of rows and column indices + """ + Input: + @param array: an array of floats which may have null-valued entries + Output: + array with nans filled in with the mean of the dataset + """ + # returns an array of rows and column indices + indices = np.where(np.isnan(array)) + + # iterate through entries which have nan values for row, col in zip(*indices): - array[row,col] = np.mean(array[~np.isnan(array[:,col]), col]) + array[row, col] = np.mean(array[~np.isnan(array[:, col]), col]) + return array def get_data(variable, feature_columns, query): - columns = ','.join(['array_agg("{col}") as "{col}"'.format(col=col) for col in feature_columns]) - data = plpy.execute('''select array_agg("{variable}") as target, {columns} from ({query}) as a'''.format( + """ + Fetch data from the database, clean, and package into + numpy arrays + Input: + @param variable: name of the target variable + @param feature_columns: list of column names + @param query: subquery that data is pulled from for the packaging + Output: + prepared data, packaged into NumPy arrays + """ + + columns = ','.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in feature_columns]) + + data = plpy.execute(''' + SELECT array_agg("{variable}") As target, + {columns} + FROM ({query}) As a + '''.format( variable = variable, columns = columns, query = query )) + target = np.array(data[0]['target']) + + # put arrays into an n x m array of arrays features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) - return replace_nan_with_mean(target), replace_nan_with_mean(features) + return replace_nan_with_mean(target), + replace_nan_with_mean(features) +# High level interface +# -------------------- + +def create_and_predict_segment_agg(target, features, target_features, target_ids, model_parameters): + """ + + """ -def create_and_predict_segment_agg(target, features, target_features, target_ids,model_parameters): clean_target = replace_nan_with_mean(target) clean_features = replace_nan_with_mean(features) target_features = replace_nan_with_mean(target_features) @@ -41,13 +78,13 @@ def create_and_predict_segment_agg(target, features, target_features, target_ids -def create_and_predict_segment(query,variable,target_query,model_params): +def create_and_predict_segment(query, variable, target_query, model_params): """ generate a segment with machine learning Stuart Lynn """ - columns = plpy.execute('select * from ({query}) a limit 1 '.format(query=query))[0].keys() + columns = plpy.execute('SELECT * FROM ({query}) As a LIMIT 1 '.format(query=query))[0].keys() feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) target,features = get_data(variable, feature_columns, query) @@ -57,46 +94,64 @@ def create_and_predict_segment(query,variable,target_query,model_params): return zip(cartodb_ids, result, np.full(result.shape, accuracy )) -def train_model(target,features,model_params,test_split): +def train_model(target, features, model_params, test_split): + """ + + """ features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) model = GradientBoostingRegressor(**model_params) model.fit(features_train, target_train) accuracy = calculate_model_accuracy(model,features,target) return model, accuracy -def calculate_model_accuracy(model,features,target): +def calculate_model_accuracy(model, features, target): + """ + Calculate the mean squared error of the model prediction + Input: + @param model: model trained from input features + @param features: features to make a prediction from + @param target: target to compare prediction to + Output: + mean squared error of the model prection compared to the target + """ prediction = model.predict(features) - return metrics.mean_squared_error(prediction,target) + return metrics.mean_squared_error(prediction, target) -def predict_segment(model,features,target_query): +def predict_segment(model, features, target_query): """ predict a segment with machine learning Stuart Lynn + + description of params? """ batch_size = 1000 - joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) + joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) - cursor = plpy.cursor('select Array[{joined_features}] features from ({target_query}) a'.format( + cursor = plpy.cursor(''' + SELECT Array[{joined_features}] As features + FROM ({target_query}) As a + '''.format( joined_features=joined_features, - target_query= target_query - )) + target_query= target_query)) results = [] while True: - rows = cursor.fetch(batch_size) + rows = cursor.fetch(batch_size) if not rows: break - batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) + batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) #Need to fix this. Should be global mean. This will cause weird effects - batch = replace_nan_with_mean(batch) - prediction = model.predict(batch) + batch = replace_nan_with_mean(batch) + prediction = model.predict(batch) results.append(prediction) - cartodb_ids = plpy.execute('select array_agg(cartodb_id order by cartodb_id) as cartodb_ids from ({0}) a '.format(target_query))[0]['cartodb_ids'] - return cartodb_ids, np.concatenate(results) - + cartodb_ids = plpy.execute(''' + SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids + FROM ({0}) As a + '''.format(target_query))[0]['cartodb_ids'] + return cartodb_ids, np.concatenate(results) From a5e4ae99ce4ecf1fa2b2970c7a63ac8f3f296c30 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 09:57:23 -0400 Subject: [PATCH 115/183] update tests --- src/pg/test/sql/06_segmentation_test.sql | 4 ++-- src/py/crankshaft/test/test_segmentation.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pg/test/sql/06_segmentation_test.sql b/src/pg/test/sql/06_segmentation_test.sql index fb6bd2b..2e93afa 100644 --- a/src/pg/test/sql/06_segmentation_test.sql +++ b/src/pg/test/sql/06_segmentation_test.sql @@ -1,5 +1,5 @@ \pset format unaligned -\set ECHO all +\set ECHO none \i test/fixtures/ml_values.sql SELECT cdb_crankshaft._cdb_random_seeds(1234); -SELECT prediction from cdb_crankshaft.CDB_CreateAndPredictSegment('select target,x1,x2,x3 from ml_values where class= $$train$$','target','select cartodb_id, target,x1,x2,x3 from ml_values where class=$$test$$') limit 20 +SELECT prediction FROM cdb_crankshaft.CDB_CreateAndPredictSegment('SELECT target, x1, x2, x3 FROM ml_values WHERE class = $$train$$','target','SELECT cartodb_id, target, x1, x2, x3 FROM ml_values WHERE class = $$test$$') LIMIT 20; diff --git a/src/py/crankshaft/test/test_segmentation.py b/src/py/crankshaft/test/test_segmentation.py index 2fc0a2d..7e806ff 100644 --- a/src/py/crankshaft/test/test_segmentation.py +++ b/src/py/crankshaft/test/test_segmentation.py @@ -23,6 +23,11 @@ class SegmentationTest(unittest.TestCase): else: return [dict( zip(['x1','x2','x3','target', 'cartodb_id'],[x1,x2,x3,y,cartodb_id]))] + def test_replace_nan_with_mean(self): + test_array = np.array([1.2, np.nan, 3.2, np.nan, np.nan]) + + + def test_create_and_predict_segment(self): n_samples = 1000 From a2b3733a1e52ef2b577ce01b18026e96de203500 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 09:58:42 -0400 Subject: [PATCH 116/183] fix improper line drop --- src/py/crankshaft/crankshaft/segmentation/segmentation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index da81842..d19eefd 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -57,8 +57,7 @@ def get_data(variable, feature_columns, query): # put arrays into an n x m array of arrays features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) - return replace_nan_with_mean(target), - replace_nan_with_mean(features) + return replace_nan_with_mean(target), replace_nan_with_mean(features) # High level interface # -------------------- From 3e01470aa001cb2aedca3b66cbe2644e58d9c0f1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 10:07:22 -0400 Subject: [PATCH 117/183] add error catching --- .../crankshaft/segmentation/segmentation.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index d19eefd..5f11e04 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -40,17 +40,15 @@ def get_data(variable, feature_columns, query): prepared data, packaged into NumPy arrays """ - columns = ','.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in feature_columns]) + columns = ','.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in feature_columns]) - data = plpy.execute(''' - SELECT array_agg("{variable}") As target, - {columns} - FROM ({query}) As a - '''.format( - variable = variable, - columns = columns, - query = query - )) + try: + data = plpy.execute('''SELECT array_agg("{variable}") As target, {columns} FROM ({query}) As a'''.format( + variable=variable, + columns=columns, + query=query)) + except Exception, e: + plpy.error('failed to fetch data to construct model') target = np.array(data[0]['target']) From c799f4d73bcc0ac6a406d5f7ac9e3d2d454c7b24 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 10:09:13 -0400 Subject: [PATCH 118/183] fix formatting of query --- src/py/crankshaft/crankshaft/segmentation/segmentation.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 5f11e04..c43ca82 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -146,9 +146,6 @@ def predict_segment(model, features, target_query): results.append(prediction) - cartodb_ids = plpy.execute(''' - SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids - FROM ({0}) As a - '''.format(target_query))[0]['cartodb_ids'] + cartodb_ids = plpy.execute('''SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids FROM ({0}) As a'''.format(target_query))[0]['cartodb_ids'] return cartodb_ids, np.concatenate(results) From 99dc363c7d1e36f7f1312940715fd4a2f5f1106f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 27 Jun 2016 10:12:06 -0400 Subject: [PATCH 119/183] format query --- src/py/crankshaft/crankshaft/segmentation/segmentation.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index c43ca82..ba103ef 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -125,10 +125,7 @@ def predict_segment(model, features, target_query): batch_size = 1000 joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) - cursor = plpy.cursor(''' - SELECT Array[{joined_features}] As features - FROM ({target_query}) As a - '''.format( + cursor = plpy.cursor('SELECT Array[{joined_features}] As features FROM ({target_query}) As a'.format( joined_features=joined_features, target_query= target_query)) From 2bb2e60af8e656e23ce21d6d96fc2c4f2c85cada Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 28 Jun 2016 10:06:54 -0400 Subject: [PATCH 120/183] catching errors --- .../crankshaft/segmentation/segmentation.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index ba103ef..c5990f8 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -48,11 +48,12 @@ def get_data(variable, feature_columns, query): columns=columns, query=query)) except Exception, e: - plpy.error('failed to fetch data to construct model') + plpy.error('Failed to access data to build segmentation model: %s' % e) + # extract target data from plpy object target = np.array(data[0]['target']) - # put arrays into an n x m array of arrays + # put n feature data arrays into an n x m array of arrays features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) return replace_nan_with_mean(target), replace_nan_with_mean(features) @@ -69,7 +70,7 @@ def create_and_predict_segment_agg(target, features, target_features, target_ids clean_features = replace_nan_with_mean(features) target_features = replace_nan_with_mean(target_features) - model, accuracy = train_model(clean_target,clean_features, model_parameters, 0.2) + model, accuracy = train_model(clean_target, clean_features, model_parameters, 0.2) prediction = model.predict(target_features) return zip(target_ids, prediction, np.full(prediction.shape, accuracy)) @@ -81,14 +82,21 @@ def create_and_predict_segment(query, variable, target_query, model_params): Stuart Lynn """ - columns = plpy.execute('SELECT * FROM ({query}) As a LIMIT 1 '.format(query=query))[0].keys() + ## fetch column names + try: + columns = plpy.execute('SELECT * FROM ({query}) As a LIMIT 1 '.format(query=query))[0].keys() + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + ## extract column names to be used in building the segmentation model feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) - target,features = get_data(variable, feature_columns, query) - model, accuracy = train_model(target,features, model_params, 0.2) - cartodb_ids, result = predict_segment(model,feature_columns,target_query) - return zip(cartodb_ids, result, np.full(result.shape, accuracy )) + ## get data from database + target, features = get_data(variable, feature_columns, query) + + model, accuracy = train_model(target, features, model_params, 0.2) + cartodb_ids, result = predict_segment(model, feature_columns, target_query) + return zip(cartodb_ids, result, np.full(result.shape, accuracy)) def train_model(target, features, model_params, test_split): @@ -98,7 +106,7 @@ def train_model(target, features, model_params, test_split): features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) model = GradientBoostingRegressor(**model_params) model.fit(features_train, target_train) - accuracy = calculate_model_accuracy(model,features,target) + accuracy = calculate_model_accuracy(model, features, target) return model, accuracy def calculate_model_accuracy(model, features, target): @@ -125,9 +133,12 @@ def predict_segment(model, features, target_query): batch_size = 1000 joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) - cursor = plpy.cursor('SELECT Array[{joined_features}] As features FROM ({target_query}) As a'.format( - joined_features=joined_features, - target_query= target_query)) + try: + cursor = plpy.cursor('SELECT Array[{joined_features}] As features FROM ({target_query}) As a'.format( + joined_features=joined_features, + target_query=target_query)) + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) results = [] @@ -142,7 +153,9 @@ def predict_segment(model, features, target_query): prediction = model.predict(batch) results.append(prediction) - - cartodb_ids = plpy.execute('''SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids FROM ({0}) As a'''.format(target_query))[0]['cartodb_ids'] + try: + cartodb_ids = plpy.execute('''SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids FROM ({0}) As a'''.format(target_query))[0]['cartodb_ids'] + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) return cartodb_ids, np.concatenate(results) From 045fd67f47db0a7f98ba2260b7e927338cb8cbb6 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 28 Jun 2016 10:15:31 -0400 Subject: [PATCH 121/183] formatting --- src/py/crankshaft/test/mock_plpy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py index c849ec4..a492c6a 100644 --- a/src/py/crankshaft/test/mock_plpy.py +++ b/src/py/crankshaft/test/mock_plpy.py @@ -2,7 +2,7 @@ import re class MockCursor: def __init__(self, data): - self.cursor_pos =0 + self.cursor_pos = 0 self.data = data def fetch(self, batch_size): @@ -38,7 +38,7 @@ class MockPlPy: def info(self, msg): self.infos.append(msg) - def cursor(self,query): + def cursor(self, query): data = self.execute(query) return MockCursor(data) From 32515f445eb231038eab2eb53fec568f8f270bb2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 28 Jun 2016 10:48:10 -0400 Subject: [PATCH 122/183] update tests and paths --- src/pg/sql/11_markov.sql | 1 - src/pg/test/sql/05_markov_test.sql | 18 +++++++++--------- .../crankshaft/space_time_dynamics/__init__.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index af5f8c8..1124abd 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -21,7 +21,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.space_time_dynamics import spatial_markov_trend diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index ad2e053..46b1422 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,21 +10,21 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, - CASE WHEN m1.cartodb_id = 1 THEN (m2.trend - 0.0714285714286) / 0.0714285714286 < 0.01 - WHEN m1.cartodb_id = 2 THEN (m2.trend - 0.222222222222) / 0.222222222222 < 0.01 - WHEN m1.cartodb_id = 3 THEN (m2.trend - 0.0526315789474) / 0.0526315789474 < 0.01 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.0714285714286) / 0.0714285714286 < 0.05 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.222222222222) / 0.222222222222 < 0.05 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.0526315789474) / 0.0526315789474 < 0.05 ELSE NULL END As trend_test, - CASE WHEN m1.cartodb_id = 1 THEN (m2.trend_up - 0.0666666666667) / 0.0666666666667 < 0.01 - WHEN m1.cartodb_id = 2 THEN (m2.trend_up - 0.181818181818) / 0.181818181818 < 0.01 - WHEN m1.cartodb_id = 3 THEN (m2.trend_up - 0.05) / 0.05 < 0.01 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.0666666666667) / 0.0666666666667 < 0.05 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.181818181818) / 0.181818181818 < 0.05 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend_up - 0.05) / 0.05 < 0.05 ELSE NULL END As trend_up_test, CASE WHEN m1.cartodb_id = 1 THEN m2.trend_down = 0.0 WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0 WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0 ELSE NULL END As trend_down_test, - CASE WHEN m1.cartodb_id = 1 THEN (m2.volatility - 0.367574633389) / 0.367574633389 < 0.1 - WHEN m1.cartodb_id = 2 THEN (m2.volatility - 0.317010832258) / 0.317010832258 < 0.1 - WHEN m1.cartodb_id = 3 THEN (m2.volatility - 0.37549966711) / 0.37549966711 < 0.1 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.volatility - 0.367574633389) / 0.367574633389 < 0.1 + WHEN m1.cartodb_id = 2 THEN abs(m2.volatility - 0.317010832258) / 0.317010832258 < 0.1 + WHEN m1.cartodb_id = 3 THEN abs(m2.volatility - 0.37549966711) / 0.37549966711 < 0.1 ELSE NULL END As volatility_test FROM markov_usjoin_example As m1 JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 diff --git a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py index a9810a7..a439286 100644 --- a/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py +++ b/src/py/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -1,2 +1,2 @@ """Import all functions from clustering libraries.""" -from crankshaft.space_time_dynamics.markov import * +from markov import * From f6f9d6e9c8ea038fc6e8b1aa25ff238aafb1b017 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 28 Jun 2016 15:49:34 +0000 Subject: [PATCH 123/183] resolves testing issues --- src/pg/test/expected/06_segmentation_test.out | 3 --- src/py/crankshaft/crankshaft/segmentation/segmentation.py | 1 - src/py/crankshaft/test/test_segmentation.py | 6 ++---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/pg/test/expected/06_segmentation_test.out b/src/pg/test/expected/06_segmentation_test.out index aa8944a..069b13b 100644 --- a/src/pg/test/expected/06_segmentation_test.out +++ b/src/pg/test/expected/06_segmentation_test.out @@ -1,7 +1,4 @@ \pset format unaligned -\set ECHO all -\i test/fixtures/ml_values.sql -SET client_min_messages TO WARNING; \set ECHO none _cdb_random_seeds diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index c5990f8..8caf055 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -90,7 +90,6 @@ def create_and_predict_segment(query, variable, target_query, model_params): ## extract column names to be used in building the segmentation model feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) - ## get data from database target, features = get_data(variable, feature_columns, query) diff --git a/src/py/crankshaft/test/test_segmentation.py b/src/py/crankshaft/test/test_segmentation.py index 7e806ff..d02e8b1 100644 --- a/src/py/crankshaft/test/test_segmentation.py +++ b/src/py/crankshaft/test/test_segmentation.py @@ -26,8 +26,6 @@ class SegmentationTest(unittest.TestCase): def test_replace_nan_with_mean(self): test_array = np.array([1.2, np.nan, 3.2, np.nan, np.nan]) - - def test_create_and_predict_segment(self): n_samples = 1000 @@ -36,6 +34,7 @@ class SegmentationTest(unittest.TestCase): training_data = self.generate_random_data(n_samples, random_state_train) test_data, test_y = self.generate_random_data(n_samples, random_state_test, row_type=True) + ids = [{'cartodb_ids': range(len(test_data))}] rows = [{'x1': 0,'x2':0,'x3':0,'y':0,'cartodb_id':0}] @@ -44,7 +43,6 @@ class SegmentationTest(unittest.TestCase): plpy._define_result('select array_agg\(cartodb\_id order by cartodb\_id\) as cartodb_ids from \(.*\) a',ids) plpy._define_result('.*select \* from test.*' ,test_data) - model_parameters = {'n_estimators': 1200, 'max_depth': 3, 'subsample' : 0.5, @@ -53,7 +51,7 @@ class SegmentationTest(unittest.TestCase): result = segmentation.create_and_predict_segment( 'select * from training', - 'y', + 'target', 'select * from test', model_parameters) From 7d6148456e512727728ce5700e2237569474ca73 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 28 Jun 2016 16:02:06 +0000 Subject: [PATCH 124/183] adding inline documentation --- .../crankshaft/segmentation/segmentation.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 8caf055..91444dd 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -63,7 +63,14 @@ def get_data(variable, feature_columns, query): def create_and_predict_segment_agg(target, features, target_features, target_ids, model_parameters): """ + Version of create_and_predict_segment that works on arrays that come stright form the SQL calling + the function. + Input: + @param target: The 1D array of lenth NSamples containing the target variable we want the model to predict + @param features: Thw 2D array of size NSamples * NFeatures that form the imput to the model + @param target_ids: A 1D array of target_ids that will be used to associate the results of the prediction with the rows which they come from + @param model_parameters: A dictionary containing parameters for the model. """ clean_target = replace_nan_with_mean(target) @@ -100,7 +107,13 @@ def create_and_predict_segment(query, variable, target_query, model_params): def train_model(target, features, model_params, test_split): """ - + Train the Gradient Boosting model on the provided data and calculate the accuracy of the model + Input: + @param target: 1D Array of the variable that the model is to be trianed to predict + @param features: 2D Array NSamples * NFeatures to use in trining the model + @param model_params: A dictionary of model parameters, the full specification can be found on the + scikit learn page for [GradientBoostingRegressor](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) + @parma test_split: The fraction of the data to be withheld for testing the model / calculating the accuray """ features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) model = GradientBoostingRegressor(**model_params) @@ -123,10 +136,11 @@ def calculate_model_accuracy(model, features, target): def predict_segment(model, features, target_query): """ - predict a segment with machine learning - Stuart Lynn - - description of params? + Use the provided model to predict the values for the new feature set + Input: + @param model: The pretrained model + @features: A list of features to use in the model prediction (list of column names) + @target_query: The query to run to obtain the data to predict on and the cartdb_ids associated with it. """ batch_size = 1000 From 5f98735ce97bb040d1233252e2fce40a175de728 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 28 Jun 2016 16:04:23 +0000 Subject: [PATCH 125/183] adding documentation for pyagg helper --- doc/04_pyAgg.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/04_pyAgg.md diff --git a/doc/04_pyAgg.md b/doc/04_pyAgg.md new file mode 100644 index 0000000..95aded9 --- /dev/null +++ b/doc/04_pyAgg.md @@ -0,0 +1,23 @@ +## PyAgg Helper Function + +### CDB_pyAgg (columns Numeric[]) + +Currently it's not possible to pass a multidiemensional array between plpsql and plpythonu. This function aims to +help fix that by aggergating the columns provided in the argument across rows in to a rows * columns + 1 length 1D array. The first element of the array is the array\_length of the columns argument so that python can reconstruct +the 2D array. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| columns | NUMERIC[] | The columns to aggregate across rows| + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| result | NUMERIC[] | An columns * rows + 1 array where the first entry is the no of columns| + + From 068c80f369dec4280cb1b6932aa6d57691836bbe Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 28 Jun 2016 18:03:52 +0000 Subject: [PATCH 126/183] fixing errors in documentatio --- doc/12_segmentation.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/12_segmentation.md b/doc/12_segmentation.md index 2404730..b6b0c95 100644 --- a/doc/12_segmentation.md +++ b/doc/12_segmentation.md @@ -50,6 +50,12 @@ This function trains a [Gradient Boosting](http://scikit-learn.org/stable/module | train\_features| numeric[] | 1D array of length n features \* n\_rows + 1 with the first entry in the array being the number of features in each row. These are the features the model will be trained on. CDB\_Crankshaft.CDB_pyAgg(Array[feature1, feature2, feature3]::numeric[]) can be used to construct this. | | prediction\_features | numeric[] | 1D array of length nfeatures\* n\_rows\_ + 1 with the first entry in the array being the number of features in each row. These are the features that will be used to predict the target variable CDB\_Crankshaft.CDB\_pyAgg(Array[feature1, feature2, feature3]::numeric[]) can be used to construct this. | | prediction\_ids | numeric[] | 1D array of length n\_rows with the ids that can use used to re-join the data with inputs | +| n\_estimators (optional) | INTEGER DEFAULT 1200 | Number of estimators to be used | +| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | +| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample parameter for GradientBooster| +| learning\_rate (optional) | DOUBLE PRECISION DEFAULT 0.01 | Learning rate for the GradientBooster | +| min\_samples\_leaf (optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf | + #### Returns @@ -60,11 +66,6 @@ A table with the following columns. | cartodb\_id | INTEGER | The CartoDB id of the row in the target\_query | | prediction | NUMERIC | The predicted value of the variable of interest | | accuracy | NUMERIC | The mean squared accuracy of the model. | -| n\_estimators (optional) | INTEGER DEFAULT 1200 | Number of estimators to be used | -| max\_depth (optional) | INTEGER DEFAULT 3 | Max tree depth | -| subsample (optional) | DOUBLE PRECISION DEFAULT 0.5 | Subsample parameter for GradientBooster| -| learning\_rate (optional) | DOUBLE PRECISION DEFAULT 0.01 | Learning rate for the GradientBooster | -| min\_samples\_leaf (optional) | INTEGER DEFAULT 1 | Minimum samples to use per leaf | #### Example Usage From 75d97915d68f1142069a8f2291326d88d31dd1f4 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 28 Jun 2016 14:23:36 -0400 Subject: [PATCH 127/183] update tests from production machine --- src/pg/test/sql/05_markov_test.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 46b1422..452fdc7 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,21 +10,21 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, - CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.0714285714286) / 0.0714285714286 < 0.05 - WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.222222222222) / 0.222222222222 < 0.05 - WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.0526315789474) / 0.0526315789474 < 0.05 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 ELSE NULL END As trend_test, - CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.0666666666667) / 0.0666666666667 < 0.05 - WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.181818181818) / 0.181818181818 < 0.05 - WHEN m1.cartodb_id = 3 THEN abs(m2.trend_up - 0.05) / 0.05 < 0.05 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.13157894736842105) / 0.13157894736842105 < 0.05 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05 ELSE NULL END As trend_up_test, CASE WHEN m1.cartodb_id = 1 THEN m2.trend_down = 0.0 WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0 WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0 ELSE NULL END As trend_down_test, CASE WHEN m1.cartodb_id = 1 THEN abs(m2.volatility - 0.367574633389) / 0.367574633389 < 0.1 - WHEN m1.cartodb_id = 2 THEN abs(m2.volatility - 0.317010832258) / 0.317010832258 < 0.1 - WHEN m1.cartodb_id = 3 THEN abs(m2.volatility - 0.37549966711) / 0.37549966711 < 0.1 + WHEN m1.cartodb_id = 2 THEN abs(m2.volatility - 0.33807340742635211) / 0.33807340742635211 < 0.1 + WHEN m1.cartodb_id = 3 THEN abs(m2.volatility - 0.3682585596149513) / 0.3682585596149513 < 0.1 ELSE NULL END As volatility_test FROM markov_usjoin_example As m1 JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2 From 476ec04386a4f0ef88f0b9a0dd2fd98a2c128826 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 28 Jun 2016 14:44:17 -0400 Subject: [PATCH 128/183] fill in rest of docs descriptions --- doc/04_markov.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/04_markov.md b/doc/04_markov.md index 880b632..a45df59 100644 --- a/doc/04_markov.md +++ b/doc/04_markov.md @@ -1,6 +1,6 @@ ## Spatial Markov -### CDB_SpatialMarkov(subquery text, column_names text array) +### CDB_SpatialMarkovTrend(subquery text, column_names text array) This function takes time series data associated with geometries and outputs likelihoods that the next value of a geometry will move up, down, or stay static as compared to the most recent measurement. For more information, read about [Spatial Dynamics in PySAL](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/dynamics.html). @@ -23,9 +23,9 @@ A table with the following columns. | Column Name | Type | Description | |-------------|------|-------------| -| trend | NUMERIC | | -| trend_up | NUMERIC | | -| trend_down | NUMERIC | The statistical significance (from 0 to 1) of a cluster or outlier classification. Lower numbers are more significant. | +| trend | NUMERIC | The probability that the measure at this location will move up (a positive number) or down (a negative number) | +| trend_up | NUMERIC | The probability that a measure will move up in subsequent steps of time | +| trend_down | NUMERIC | The probability that a measure will move down in subsequent steps of time | | volatility | NUMERIC | A measure of the variance of the probabilities returned from the Spatial Markov predictions | | rowid | NUMERIC | id of the row that corresponds to the `id_col` (by default `cartodb_id` of the input rows) | @@ -34,13 +34,14 @@ A table with the following columns. ```sql SELECT + c.cartodb_id, c.the_geom, m.trend, m.trend_up, m.trend_down, m.volatility -FROM CDB_SpatialMarkov('SELECT * FROM nyc_real_estate' - Array['m03y2009','m03y2010','m03y2011','m03y2012','m03y2013','m03y2014','m03y2015','m03y2016']) As m +FROM CDB_SpatialMarkovTrend('SELECT * FROM nyc_real_estate' + Array['m03y2009','m03y2010','m03y2011','m03y2012','m03y2013','m03y2014','m03y2015','m03y2016']) As m JOIN nyc_real_estate As c ON c.cartodb_id = m.rowid; ``` From 3a2e5ec7f9dac1926095226ebcb1e167c81016d2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 16:06:34 +0200 Subject: [PATCH 129/183] Fix Markov trend_test that expects a bool #77 --- src/pg/test/sql/05_markov_test.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index 452fdc7..b2891f1 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,9 +10,9 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, - CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 - WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 - WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.01 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 < 0.01 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.01 ELSE NULL END As trend_test, CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05 WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.13157894736842105) / 0.13157894736842105 < 0.05 From 9e4d378a0890a7c60e886782d5373d6e84408304 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 29 Jun 2016 10:27:19 -0400 Subject: [PATCH 130/183] updates error bounds --- src/pg/test/sql/05_markov_test.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/test/sql/05_markov_test.sql b/src/pg/test/sql/05_markov_test.sql index b2891f1..2d7e667 100644 --- a/src/pg/test/sql/05_markov_test.sql +++ b/src/pg/test/sql/05_markov_test.sql @@ -10,9 +10,9 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT m1.cartodb_id, - CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.01 - WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 < 0.01 - WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.01 + CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05 + WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 < 0.05 + WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05 ELSE NULL END As trend_test, CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05 WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.13157894736842105) / 0.13157894736842105 < 0.05 From cfb58f78987f388ab0e0b380ced015993ec3af94 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 29 Jun 2016 14:46:03 +0000 Subject: [PATCH 131/183] removing full function calls to be compatiable with numpy 1.6.1 --- src/py/crankshaft/crankshaft/segmentation/segmentation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/segmentation/segmentation.py b/src/py/crankshaft/crankshaft/segmentation/segmentation.py index 91444dd..ed61139 100644 --- a/src/py/crankshaft/crankshaft/segmentation/segmentation.py +++ b/src/py/crankshaft/crankshaft/segmentation/segmentation.py @@ -79,7 +79,8 @@ def create_and_predict_segment_agg(target, features, target_features, target_ids model, accuracy = train_model(clean_target, clean_features, model_parameters, 0.2) prediction = model.predict(target_features) - return zip(target_ids, prediction, np.full(prediction.shape, accuracy)) + accuracy_array = [accuracy]*prediction.shape[0] + return zip(target_ids, prediction, np.full(prediction.shape, accuracy_array)) @@ -102,7 +103,8 @@ def create_and_predict_segment(query, variable, target_query, model_params): model, accuracy = train_model(target, features, model_params, 0.2) cartodb_ids, result = predict_segment(model, feature_columns, target_query) - return zip(cartodb_ids, result, np.full(result.shape, accuracy)) + accuracy_array = [accuracy]*result.shape[0] + return zip(cartodb_ids, result, accuracy_array) def train_model(target, features, model_params, test_split): From 505ae0fb4453e81d8995ff20d1ad523e9e2d85ee Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 17:02:15 +0200 Subject: [PATCH 132/183] Go back to using numpy==1.6.1 --- src/py/crankshaft/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index de6aeee..cd8ad99 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,8 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['joblib==0.8.3', 'numpy==1.11.0', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + # IMPORTANT NOTE: please don't change this line. Instead issue a ticket to systems for evaluation. + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], requires=['pysal', 'numpy', 'sklearn'], From cb330ebe6b6fe3148502926d2d85124014cbd530 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 18:12:21 +0200 Subject: [PATCH 133/183] Make test more robust --- src/pg/test/expected/06_segmentation_test.out | 42 +++++++++---------- src/pg/test/sql/06_segmentation_test.sql | 30 ++++++++++++- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/pg/test/expected/06_segmentation_test.out b/src/pg/test/expected/06_segmentation_test.out index 069b13b..a4c17a9 100644 --- a/src/pg/test/expected/06_segmentation_test.out +++ b/src/pg/test/expected/06_segmentation_test.out @@ -3,25 +3,25 @@ _cdb_random_seeds (1 row) -prediction -4.5656517130822492 -1.7928053473230694 -1.0283378773916563 -2.6586517814904593 -2.9699056242935944 -3.9550646059951347 -4.1662572444459745 -3.8126334839264162 -1.8809821053623488 -1.6349065129019873 -3.0391288591472954 -3.3035970359672553 -1.5835471589451968 -3.7530378537263638 -1.0833589653009252 -3.8104965452882897 -2.665217959294802 -1.5850334252802472 -3.679401198805563 -3.5332033186588636 +within_tolerance +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t +t (20 rows) diff --git a/src/pg/test/sql/06_segmentation_test.sql b/src/pg/test/sql/06_segmentation_test.sql index 2e93afa..932cb04 100644 --- a/src/pg/test/sql/06_segmentation_test.sql +++ b/src/pg/test/sql/06_segmentation_test.sql @@ -2,4 +2,32 @@ \set ECHO none \i test/fixtures/ml_values.sql SELECT cdb_crankshaft._cdb_random_seeds(1234); -SELECT prediction FROM cdb_crankshaft.CDB_CreateAndPredictSegment('SELECT target, x1, x2, x3 FROM ml_values WHERE class = $$train$$','target','SELECT cartodb_id, target, x1, x2, x3 FROM ml_values WHERE class = $$test$$') LIMIT 20; + +WITH expected AS ( + SELECT generate_series(1000,1020) AS id, unnest(ARRAY[ + 4.5656517130822492, + 1.7928053473230694, + 1.0283378773916563, + 2.6586517814904593, + 2.9699056242935944, + 3.9550646059951347, + 4.1662572444459745, + 3.8126334839264162, + 1.8809821053623488, + 1.6349065129019873, + 3.0391288591472954, + 3.3035970359672553, + 1.5835471589451968, + 3.7530378537263638, + 1.0833589653009252, + 3.8104965452882897, + 2.665217959294802, + 1.5850334252802472, + 3.679401198805563, + 3.5332033186588636 + ]) AS expected LIMIT 20 +), prediction AS ( + SELECT cartodb_id::integer id, prediction + FROM cdb_crankshaft.CDB_CreateAndPredictSegment('SELECT target, x1, x2, x3 FROM ml_values WHERE class = $$train$$','target','SELECT cartodb_id, target, x1, x2, x3 FROM ml_values WHERE class = $$test$$') + LIMIT 20 +) SELECT abs(e.expected - p.prediction) <= 1e-9 AS within_tolerance FROM expected e, prediction p WHERE e.id = p.id; From d97231f6044b070747fd9f0276ad4d4c4dd02ed1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 18:34:01 +0200 Subject: [PATCH 134/183] Revert changes in release/ dir --- release/crankshaft--0.0.1.sql | 27 ------------------------ release/python/0.0.1/crankshaft/setup.py | 3 ++- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/release/crankshaft--0.0.1.sql b/release/crankshaft--0.0.1.sql index c72e5fc..436beea 100644 --- a/release/crankshaft--0.0.1.sql +++ b/release/crankshaft--0.0.1.sql @@ -137,33 +137,6 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE; -CREATE OR REPLACE FUNCTION - cdb_create_segment ( - segment_name TEXT, - table_name TEXT, - column_name TEXT, - geoid_column TEXT DEFAULT 'geoid', - census_table TEXT DEFAULT 'block_groups' - ) -RETURNS NUMERIC -AS $$ - from crankshaft import segmentation - # TODO: use named parameters or a dictionary - return segmentation.create_segment(segment_name,table_name,column_name,geoid_column,census_table,'random_forest') -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION - cdb_predict_segment ( - segment_name TEXT, - geoid_column TEXT DEFAULT 'geoid', - census_table TEXT DEFAULT 'block_groups' - ) -RETURNS TABLE(geoid TEXT, prediction NUMERIC) -AS $$ - from crankshaft.segmentation import create_segemnt - # TODO: use named parameters or a dictionary - return create_segment('table') -$$ LANGUAGE plpythonu; -- Make sure by default there are no permissions for publicuser -- NOTE: this happens at extension creation time, as part of an implicit transaction. -- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; diff --git a/release/python/0.0.1/crankshaft/setup.py b/release/python/0.0.1/crankshaft/setup.py index 798a96d..f045b62 100644 --- a/release/python/0.0.1/crankshaft/setup.py +++ b/release/python/0.0.1/crankshaft/setup.py @@ -40,8 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.11.0','numpy==1.10.1','scipy==0.17.0','pandas','sklearn'], + install_requires=['pysal==1.11.0','numpy==1.6.1','scipy==0.17.0'], + requires=['pysal', 'numpy'], test_suite='test' ) From 76ee4cacbcc67b51bfd42b894042d6b0eee9d908 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 18:49:14 +0200 Subject: [PATCH 135/183] Revert changes in release/ dir --- release/python/0.0.1/crankshaft/crankshaft/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/release/python/0.0.1/crankshaft/crankshaft/__init__.py b/release/python/0.0.1/crankshaft/crankshaft/__init__.py index bc8e065..d07e330 100644 --- a/release/python/0.0.1/crankshaft/crankshaft/__init__.py +++ b/release/python/0.0.1/crankshaft/crankshaft/__init__.py @@ -1,3 +1,2 @@ import random_seeds import clustering -import segmentation From 702a1fb1ede06fa23fc75021643354da4c970b51 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 19:00:53 +0200 Subject: [PATCH 136/183] Update NEWS.md --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index c011a0d..8d40482 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +0.0.5 (2016-06-29) +------------------ +* Adds Spatial Markov function +* Adds Spacial interpolation function +* Adds `CDB_pyAgg (columns Numeric[])` helper function +* Adds Segmentation Functions + 0.0.4 (2016-06-20) ------------------ * Remove cartodb extension dependency from tests From 408dc6806ea48cfb8a1a2f76e2d41876ad8b9046 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 19:10:26 +0200 Subject: [PATCH 137/183] Use 0.1.0 instead of 0.0.5 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8d40482..2c19570 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -0.0.5 (2016-06-29) +0.1.0 (2016-06-29) ------------------ * Adds Spatial Markov function * Adds Spacial interpolation function From 78ceb02c2277af8af38441ffa3a9ada98d549452 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 19:15:28 +0200 Subject: [PATCH 138/183] Add release files generated with make release --- release/crankshaft--0.1.0.sql | 686 ++++++++++++++++++ release/crankshaft.control | 2 +- .../0.1.0/crankshaft/crankshaft/__init__.py | 5 + .../crankshaft/clustering/__init__.py | 3 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 262 +++++++ .../crankshaft/pysal_utils/__init__.py | 2 + .../crankshaft/pysal_utils/pysal_utils.py | 188 +++++ .../crankshaft/crankshaft/random_seeds.py | 11 + .../crankshaft/segmentation/__init__.py | 1 + .../crankshaft/segmentation/segmentation.py | 176 +++++ .../space_time_dynamics/__init__.py | 2 + .../crankshaft/space_time_dynamics/markov.py | 189 +++++ release/python/0.1.0/crankshaft/setup.py | 49 ++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../crankshaft/test/fixtures/markov.json | 1 + .../0.1.0/crankshaft/test/fixtures/moran.json | 52 ++ .../crankshaft/test/fixtures/neighbors.json | 54 ++ .../test/fixtures/neighbors_markov.json | 1 + .../python/0.1.0/crankshaft/test/helper.py | 13 + .../python/0.1.0/crankshaft/test/mock_plpy.py | 52 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 + .../crankshaft/test/test_clustering_moran.py | 88 +++ .../0.1.0/crankshaft/test/test_pysal_utils.py | 142 ++++ .../crankshaft/test/test_segmentation.py | 64 ++ .../test/test_space_time_dynamics.py | 324 +++++++++ src/pg/crankshaft.control | 2 +- 27 files changed, 2424 insertions(+), 2 deletions(-) create mode 100644 release/crankshaft--0.1.0.sql create mode 100644 release/python/0.1.0/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/segmentation/__init__.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/segmentation/segmentation.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/__init__.py create mode 100644 release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/markov.py create mode 100644 release/python/0.1.0/crankshaft/setup.py create mode 100644 release/python/0.1.0/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.1.0/crankshaft/test/fixtures/markov.json create mode 100644 release/python/0.1.0/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.1.0/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.1.0/crankshaft/test/fixtures/neighbors_markov.json create mode 100644 release/python/0.1.0/crankshaft/test/helper.py create mode 100644 release/python/0.1.0/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.1.0/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.1.0/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.1.0/crankshaft/test/test_pysal_utils.py create mode 100644 release/python/0.1.0/crankshaft/test/test_segmentation.py create mode 100644 release/python/0.1.0/crankshaft/test/test_space_time_dynamics.py diff --git a/release/crankshaft--0.1.0.sql b/release/crankshaft--0.1.0.sql new file mode 100644 index 0000000..d5a5b66 --- /dev/null +++ b/release/crankshaft--0.1.0.sql @@ -0,0 +1,686 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.1.0'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION + CDB_PyAggS(current_state Numeric[], current_row Numeric[]) + returns NUMERIC[] as $$ + BEGIN + if array_upper(current_state,1) is null then + RAISE NOTICE 'setting state %',array_upper(current_row,1); + current_state[1] = array_upper(current_row,1); + end if; + return array_cat(current_state,current_row) ; + END + $$ LANGUAGE plpgsql; + + +CREATE AGGREGATE CDB_PyAgg(NUMERIC[])( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" +); + + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = {'n_estimators': n_estimators, + 'max_depth': max_depth, + 'subsample': subsample, + 'learning_rate': learning_rate, + 'min_samples_leaf': min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg(np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC) +AS $$ + from crankshaft.segmentation import create_and_predict_segment + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + return create_and_predict_segment(query,variable_name,target_table, model_params) +$$ LANGUAGE plpythonu; +-- 0: nearest neighbor +-- 1: barymetric +-- 2: IDW + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + output numeric; +BEGIN + EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs; + SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a; + + RETURN output; +END; +$$ +language plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + gs2 geometry[]; + vs2 numeric[]; + g geometry; + vertex geometry[]; + sg numeric; + sa numeric; + sb numeric; + sc numeric; + va numeric; + vb numeric; + vc numeric; + output numeric; +BEGIN + output := -999.999; + -- nearest + IF method = 0 THEN + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v) + SELECT a.v INTO output FROM a ORDER BY point<->a.g LIMIT 1; + RETURN output; + + -- barymetric + ELSIF method = 1 THEN + WITH a as (SELECT unnest(geomin) AS e), + b as (SELECT ST_DelaunayTriangles(ST_Collect(a.e),0.001, 0) AS t FROM a), + c as (SELECT (ST_Dump(t)).geom as v FROM b), + d as (SELECT v FROM c WHERE ST_Within(point, v)) + SELECT v INTO g FROM d; + IF g is null THEN + -- out of the realm of the input data + RETURN -888.888; + END IF; + -- vertex of the selected cell + WITH a AS (SELECT (ST_DumpPoints(g)).geom AS v) + SELECT array_agg(v) INTO vertex FROM a; + + -- retrieve the value of each vertex + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]); + + SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc; + + output := (coalesce(sa,0) * coalesce(va,0) + coalesce(sb,0) * coalesce(vb,0) + coalesce(sc,0) * coalesce(vc,0)) / coalesce(sg); + RETURN output; + + -- IDW + -- p1: limit the number of neighbors, 0->no limit + -- p2: order of distance decay, 0-> order 1 + ELSIF method = 2 THEN + + IF p2 = 0 THEN + p2 := 1; + END IF; + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v), + b as (SELECT a.g, a.v FROM a ORDER BY point<->a.g) + SELECT array_agg(b.g), array_agg(b.v) INTO gs, vs FROM b; + IF p1::integer>0 THEN + gs2:=gs; + vs2:=vs; + FOR i IN 1..p1 + LOOP + gs2 := gs2 || gs[i]; + vs2 := vs2 || vs[i]; + END LOOP; + ELSE + gs2:=gs; + vs2:=vs; + END IF; + + WITH a as (SELECT unnest(gs2) as g, unnest(vs2) as v), + b as ( + SELECT + (1/ST_distance(point, a.g)^p2::integer) as k, + (a.v/ST_distance(point, a.g)^p2::integer) as f + FROM a + ) + SELECT sum(b.f)/sum(b.k) INTO output FROM b; + RETURN output; + + END IF; + + RETURN -777.777; + +END; +$$ +language plpgsql IMMUTABLE; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + +CREATE OR REPLACE FUNCTION + CDB_SpatialMarkovTrend ( + subquery TEXT, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) +AS $$ + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- input table format: identical to above but in a predictable format +-- Sample function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- 'date_1') + + +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col_min text, +-- time_col_max text, +-- date_format text, -- '_YYYY_MM_DD' +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- +-- -- input table format: +-- -- id | geom | date | measurement +-- -- 1 | Pt1 | 12/3 | 13.2 +-- -- 2 | Pt2 | 11/5 | 11.3 +-- -- 3 | Pt1 | 11/13 | 12.9 +-- -- 4 | Pt3 | 12/19 | 10.1 +-- -- ... +-- +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col text, +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 01088b1..876fadc 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.4' +default_version = '0.1.0' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.1.0/crankshaft/crankshaft/__init__.py b/release/python/0.1.0/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..4e06bc5 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/__init__.py @@ -0,0 +1,5 @@ +"""Import all modules""" +import crankshaft.random_seeds +import crankshaft.clustering +import crankshaft.space_time_dynamics +import crankshaft.segmentation diff --git a/release/python/0.1.0/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.1.0/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..ed34fe0 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,3 @@ +"""Import all functions from for clustering""" +from moran import * +from kmeans import * diff --git a/release/python/0.1.0/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.1.0/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.1.0/crankshaft/crankshaft/clustering/moran.py b/release/python/0.1.0/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..3282f5f --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,262 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy +from collections import OrderedDict + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr_name), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = OrderedDict([("id_col", id_col), + ("attr1", numerator), + ("attr2", denominator) + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = OrderedDict([("id_col", id_col), + ("numerator", numerator), + ("denominator", denominator), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr1), + ("attr2", attr2), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..fdf073b --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1,2 @@ +"""Import all functions for pysal_utils""" +from crankshaft.pysal_utils.pysal_utils import * diff --git a/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..4622925 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,188 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on + translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res dict-like: query results with attributes and neighbors + """ + # if w_type.lower() == 'knn': + # row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + # weights = {x['id']: row_normed_weights for x in query_res} + # else: + # weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + # if len(x['neighbors']) > 0 + # else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + print 'len of neighbors: %d' % len(neighbors) + + built_weight = ps.W(neighbors) + built_weight.transform = 'r' + + return built_weight + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attr_string = "" + template = "i.\"%(col)s\"::numeric As attr%(alias_num)s, " + + if 'time_cols' in params: + ## if markov analysis + attrs = params['time_cols'] + + for idx, val in enumerate(attrs): + attr_string += template % {"col": val, "alias_num": idx + 1} + else: + ## if moran's analysis + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')] + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": params[val], "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Construct where conditions when building neighbors query + Create portion of WHERE clauses for weeding out NULL-valued geometries + Input: dict of params: + {'subquery': ..., + 'numerator': 'data1', + 'denominator': 'data2', + '': ...} + Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' + Input: + {'subquery': ..., + 'time_cols': ['time1', 'time2', 'time3'], + 'etc': ...} + Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT + NULL AND idx_replace."time3" IS NOT NULL' + """ + attr_string = [] + template = "idx_replace.\"%s\" IS NOT NULL" + + if 'time_cols' in params: + ## markov where clauses + attrs = params['time_cols'] + # add values to template + for attr in attrs: + attr_string.append(template % attr) + else: + ## moran where clauses + + # get keys + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]) + # add values to template + for attr in attrs: + attr_string.append(template % params[attr]) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.1.0/crankshaft/crankshaft/random_seeds.py b/release/python/0.1.0/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..31958cb --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,11 @@ +"""Random seed generator used for non-deterministic functions in crankshaft""" +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.1.0/crankshaft/crankshaft/segmentation/__init__.py b/release/python/0.1.0/crankshaft/crankshaft/segmentation/__init__.py new file mode 100644 index 0000000..b825e85 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/segmentation/__init__.py @@ -0,0 +1 @@ +from segmentation import * diff --git a/release/python/0.1.0/crankshaft/crankshaft/segmentation/segmentation.py b/release/python/0.1.0/crankshaft/crankshaft/segmentation/segmentation.py new file mode 100644 index 0000000..ed61139 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/segmentation/segmentation.py @@ -0,0 +1,176 @@ +""" +Segmentation creation and prediction +""" + +import sklearn +import numpy as np +import plpy +from sklearn.ensemble import GradientBoostingRegressor +from sklearn import metrics +from sklearn.cross_validation import train_test_split + +# Lower level functions +#---------------------- + +def replace_nan_with_mean(array): + """ + Input: + @param array: an array of floats which may have null-valued entries + Output: + array with nans filled in with the mean of the dataset + """ + # returns an array of rows and column indices + indices = np.where(np.isnan(array)) + + # iterate through entries which have nan values + for row, col in zip(*indices): + array[row, col] = np.mean(array[~np.isnan(array[:, col]), col]) + + return array + +def get_data(variable, feature_columns, query): + """ + Fetch data from the database, clean, and package into + numpy arrays + Input: + @param variable: name of the target variable + @param feature_columns: list of column names + @param query: subquery that data is pulled from for the packaging + Output: + prepared data, packaged into NumPy arrays + """ + + columns = ','.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in feature_columns]) + + try: + data = plpy.execute('''SELECT array_agg("{variable}") As target, {columns} FROM ({query}) As a'''.format( + variable=variable, + columns=columns, + query=query)) + except Exception, e: + plpy.error('Failed to access data to build segmentation model: %s' % e) + + # extract target data from plpy object + target = np.array(data[0]['target']) + + # put n feature data arrays into an n x m array of arrays + features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) + + return replace_nan_with_mean(target), replace_nan_with_mean(features) + +# High level interface +# -------------------- + +def create_and_predict_segment_agg(target, features, target_features, target_ids, model_parameters): + """ + Version of create_and_predict_segment that works on arrays that come stright form the SQL calling + the function. + + Input: + @param target: The 1D array of lenth NSamples containing the target variable we want the model to predict + @param features: Thw 2D array of size NSamples * NFeatures that form the imput to the model + @param target_ids: A 1D array of target_ids that will be used to associate the results of the prediction with the rows which they come from + @param model_parameters: A dictionary containing parameters for the model. + """ + + clean_target = replace_nan_with_mean(target) + clean_features = replace_nan_with_mean(features) + target_features = replace_nan_with_mean(target_features) + + model, accuracy = train_model(clean_target, clean_features, model_parameters, 0.2) + prediction = model.predict(target_features) + accuracy_array = [accuracy]*prediction.shape[0] + return zip(target_ids, prediction, np.full(prediction.shape, accuracy_array)) + + + +def create_and_predict_segment(query, variable, target_query, model_params): + """ + generate a segment with machine learning + Stuart Lynn + """ + + ## fetch column names + try: + columns = plpy.execute('SELECT * FROM ({query}) As a LIMIT 1 '.format(query=query))[0].keys() + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + ## extract column names to be used in building the segmentation model + feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) + ## get data from database + target, features = get_data(variable, feature_columns, query) + + model, accuracy = train_model(target, features, model_params, 0.2) + cartodb_ids, result = predict_segment(model, feature_columns, target_query) + accuracy_array = [accuracy]*result.shape[0] + return zip(cartodb_ids, result, accuracy_array) + + +def train_model(target, features, model_params, test_split): + """ + Train the Gradient Boosting model on the provided data and calculate the accuracy of the model + Input: + @param target: 1D Array of the variable that the model is to be trianed to predict + @param features: 2D Array NSamples * NFeatures to use in trining the model + @param model_params: A dictionary of model parameters, the full specification can be found on the + scikit learn page for [GradientBoostingRegressor](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) + @parma test_split: The fraction of the data to be withheld for testing the model / calculating the accuray + """ + features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) + model = GradientBoostingRegressor(**model_params) + model.fit(features_train, target_train) + accuracy = calculate_model_accuracy(model, features, target) + return model, accuracy + +def calculate_model_accuracy(model, features, target): + """ + Calculate the mean squared error of the model prediction + Input: + @param model: model trained from input features + @param features: features to make a prediction from + @param target: target to compare prediction to + Output: + mean squared error of the model prection compared to the target + """ + prediction = model.predict(features) + return metrics.mean_squared_error(prediction, target) + +def predict_segment(model, features, target_query): + """ + Use the provided model to predict the values for the new feature set + Input: + @param model: The pretrained model + @features: A list of features to use in the model prediction (list of column names) + @target_query: The query to run to obtain the data to predict on and the cartdb_ids associated with it. + """ + + batch_size = 1000 + joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) + + try: + cursor = plpy.cursor('SELECT Array[{joined_features}] As features FROM ({target_query}) As a'.format( + joined_features=joined_features, + target_query=target_query)) + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + results = [] + + while True: + rows = cursor.fetch(batch_size) + if not rows: + break + batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) + + #Need to fix this. Should be global mean. This will cause weird effects + batch = replace_nan_with_mean(batch) + prediction = model.predict(batch) + results.append(prediction) + + try: + cartodb_ids = plpy.execute('''SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids FROM ({0}) As a'''.format(target_query))[0]['cartodb_ids'] + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + return cartodb_ids, np.concatenate(results) diff --git a/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/__init__.py b/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/__init__.py new file mode 100644 index 0000000..a439286 --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -0,0 +1,2 @@ +"""Import all functions from clustering libraries.""" +from markov import * diff --git a/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/markov.py b/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/markov.py new file mode 100644 index 0000000..bbf524d --- /dev/null +++ b/release/python/0.1.0/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -0,0 +1,189 @@ +""" +Spatial dynamics measurements using Spatial Markov +""" + + +import numpy as np +import pysal as ps +import plpy +import crankshaft.pysal_utils as pu + +def spatial_markov_trend(subquery, time_cols, num_classes=7, + w_type='knn', num_ngbrs=5, permutations=0, + geom_col='the_geom', id_col='cartodb_id'): + """ + Predict the trends of a unit based on: + 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) + 2. average class of its neighbors + + Inputs: + @param subquery string: e.g., SELECT the_geom, cartodb_id, + interesting_time_column FROM table_name + @param time_cols list of strings: list of strings of column names + @param num_classes (optional): number of classes to break distribution + of values into. Currently uses quantile bins. + @param w_type string (optional): weight type ('knn' or 'queen') + @param num_ngbrs int (optional): number of neighbors (if knn type) + @param permutations int (optional): number of permutations for test + stats + @param geom_col string (optional): name of column which contains the + geometries + @param id_col string (optional): name of column which has the ids of + the table + + Outputs: + @param trend_up float: probablity that a geom will move to a higher + class + @param trend_down float: probablity that a geom will move to a lower + class + @param trend float: (trend_up - trend_down) / trend_static + @param volatility float: a measure of the volatility based on + probability stddev(prob array) + """ + + if len(time_cols) < 2: + plpy.error('More than one time column needs to be passed') + + qvals = {"id_col": id_col, + "time_cols": time_cols, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + try: + query_result = plpy.execute( + pu.construct_neighbor_query(w_type, qvals) + ) + if len(query_result) == 0: + return zip([None], [None], [None], [None], [None]) + except plpy.SPIError, err: + plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals))) + plpy.error('Query failed, check the input parameters') + return zip([None], [None], [None], [None], [None]) + + ## build weight + weights = pu.get_weight(query_result, w_type) + weights.transform = 'r' + + ## prep time data + t_data = get_time_data(query_result, time_cols) + + plpy.debug('shape of t_data %d, %d' % t_data.shape) + plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape) + plpy.debug('first num elements: %f' % t_data[0, 0]) + + sp_markov_result = ps.Spatial_Markov(t_data, + weights, + k=num_classes, + fixed=False, + permutations=permutations) + + ## get lag classes + lag_classes = ps.Quantiles( + ps.lag_spatial(weights, t_data[:, -1]), + k=num_classes).yb + + ## look up probablity distribution for each unit according to class and lag class + prob_dist = get_prob_dist(sp_markov_result.P, + lag_classes, + sp_markov_result.classes[:, -1]) + + ## find the ups and down and overall distribution of each cell + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, + sp_markov_result.classes[:, -1]) + + ## output the results + return zip(trend, trend_up, trend_down, volatility, weights.id_order) + +def get_time_data(markov_data, time_cols): + """ + Extract the time columns and bin appropriately + """ + num_attrs = len(time_cols) + return np.array([[x['attr' + str(i)] for x in markov_data] + for i in range(1, num_attrs+1)], dtype=float).transpose() + +## not currently used +def rebin_data(time_data, num_time_per_bin): + """ + Convert an n x l matrix into an (n/m) x l matrix where the values are + reduced (averaged) for the intervening states: + 1 2 3 4 1.5 3.5 + 5 6 7 8 -> 5.5 7.5 + 9 8 7 6 8.5 6.5 + 5 4 3 2 4.5 2.5 + + if m = 2, the 4 x 4 matrix is transformed to a 2 x 4 matrix. + + This process effectively resamples the data at a longer time span n + units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining + two columns are binned together as the last time period, while the + first three are binned together for the first period. + + Input: + @param time_data n x l ndarray: measurements of an attribute at + different time intervals + @param num_time_per_bin int: number of columns to average into a new + column + Output: + ceil(n / m) x l ndarray of resampled time series + """ + + if time_data.shape[1] % num_time_per_bin == 0: + ## if fit is perfect, then use it + n_max = time_data.shape[1] / num_time_per_bin + else: + ## fit remainders into an additional column + n_max = time_data.shape[1] / num_time_per_bin + 1 + + return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) + for i in range(n_max)]).T + +def get_prob_dist(transition_matrix, lag_indices, unit_indices): + """ + Given an array of transition matrices, look up the probability + associated with the arrangements passed + + Input: + @param transition_matrix ndarray[k,k,k]: + @param lag_indices ndarray: + @param unit_indices ndarray: + + Output: + Array of probability distributions + """ + + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] + for i in range(len(lag_indices))]) + +def get_prob_stats(prob_dist, unit_indices): + """ + get the statistics of the probability distributions + + Outputs: + @param trend_up ndarray(float): sum of probabilities for upward + movement (relative to the unit index of that prob) + @param trend_down ndarray(float): sum of probabilities for downward + movement (relative to the unit index of that prob) + @param trend ndarray(float): difference of upward and downward + movements + """ + + num_elements = len(unit_indices) + trend_up = np.empty(num_elements, dtype=float) + trend_down = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) + + for i in range(num_elements): + trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() + trend_down[i] = prob_dist[i, :unit_indices[i]].sum() + if prob_dist[i, unit_indices[i]] > 0.0: + trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + else: + trend[i] = None + + ## calculate volatility of distribution + volatility = prob_dist.std(axis=1) + + return trend_up, trend_down, trend, volatility diff --git a/release/python/0.1.0/crankshaft/setup.py b/release/python/0.1.0/crankshaft/setup.py new file mode 100644 index 0000000..273cce1 --- /dev/null +++ b/release/python/0.1.0/crankshaft/setup.py @@ -0,0 +1,49 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.1.0', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + # IMPORTANT NOTE: please don't change this line. Instead issue a ticket to systems for evaluation. + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.1.0/crankshaft/test/fixtures/kmeans.json b/release/python/0.1.0/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.1.0/crankshaft/test/fixtures/markov.json b/release/python/0.1.0/crankshaft/test/fixtures/markov.json new file mode 100644 index 0000000..d60e4e0 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/fixtures/markov.json @@ -0,0 +1 @@ +[[0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 0], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 1], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 2], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 3], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 4], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 5], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 6], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 7], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 8], [0.19047619047619049, 0.16, 0.0, 0.32594478059941379, 9], [-0.23529411764705882, 0.0, 0.19047619047619047, 0.31356338348865387, 10], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 11], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 12], [0.027777777777777783, 0.11111111111111112, 0.088888888888888892, 0.30339641183779581, 13], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 14], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 15], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 16], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 17], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 18], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 19], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 20], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 21], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 22], [-0.16666666666666663, 0.18181818181818182, 0.27272727272727271, 0.20246415864836445, 23], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 24], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 25], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 26], [-0.043478260869565216, 0.0, 0.041666666666666664, 0.37950991789118999, 27], [0.22222222222222221, 0.18181818181818182, 0.0, 0.31701083225750354, 28], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 29], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 30], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 31], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 32], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 33], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 34], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 35], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 36], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 37], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 38], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 39], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 40], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 41], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 42], [0.0, 0.0, 0.0, 0.40000000000000002, 43], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 44], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 45], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 46], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 47]] diff --git a/release/python/0.1.0/crankshaft/test/fixtures/moran.json b/release/python/0.1.0/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.1.0/crankshaft/test/fixtures/neighbors.json b/release/python/0.1.0/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.1.0/crankshaft/test/fixtures/neighbors_markov.json b/release/python/0.1.0/crankshaft/test/fixtures/neighbors_markov.json new file mode 100644 index 0000000..45a20e7 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/fixtures/neighbors_markov.json @@ -0,0 +1 @@ +[{"neighbors": [10, 7, 21, 23, 1], "y1995": 0.87654416055651474, "y1997": 0.85637566664752718, "y1996": 0.8631470006766887, "y1999": 0.84461540228037335, "y1998": 0.84811668329242784, "y2006": 0.86302631339545688, "y2007": 0.86148266513456728, "y2004": 0.86416611731111015, "y2005": 0.87119374831581786, "y2002": 0.85012592862683589, "y2003": 0.8550965633336135, "y2000": 0.83271652434603094, "y2001": 0.83786313566577242, "id": 0, "y2008": 0.86252252380501315, "y2009": 0.86746356478544273}, {"neighbors": [5, 7, 22, 29, 3], "y1995": 0.91889509774542122, "y1997": 0.92333257900976462, "y1996": 0.91757931190043385, "y1999": 0.92552387732371888, "y1998": 0.92517289327379471, "y2006": 0.91706053906277052, "y2007": 0.90139504820726424, "y2004": 0.89815175749309051, "y2005": 0.91832090781161113, "y2002": 0.89431990798552208, "y2003": 0.88924793576523797, "y2000": 0.90746978227271013, "y2001": 0.89830489127332913, "id": 1, "y2008": 0.87897455159080617, "y2009": 0.86216858051752643}, {"neighbors": [11, 8, 13, 18, 17], "y1995": 0.82591007476914713, "y1997": 0.81989792988843901, "y1996": 0.82548595539161707, "y1999": 0.81731522200916285, "y1998": 0.81503235035017918, "y2006": 0.81814804358939286, "y2007": 0.83675961003285626, "y2004": 0.82668195534569056, "y2005": 0.82373723764184559, "y2002": 0.80849979516360859, "y2003": 0.82258550658074148, "y2000": 0.78964559168205917, "y2001": 0.8058444152731008, "id": 2, "y2008": 0.8357419865626442, "y2009": 0.84647177436289112}, {"neighbors": [4, 14, 9, 5, 12], "y1995": 1.0908817638059434, "y1997": 1.0845641754849344, "y1996": 1.0853768890893893, "y1999": 1.098988414417104, "y1998": 1.0841540389418189, "y2006": 1.1316479722785828, "y2007": 1.1295850763954971, "y2004": 1.1139980568106316, "y2005": 1.1216802898290368, "y2002": 1.1116069731657288, "y2003": 1.1088862051501811, "y2000": 1.1450694824791507, "y2001": 1.1215113292620285, "id": 3, "y2008": 1.1137181812756343, "y2009": 1.0993677488645406}, {"neighbors": [14, 3, 9, 31, 12], "y1995": 1.1073144618319228, "y1997": 1.1328363804627946, "y1996": 1.1137394350312471, "y1999": 1.1591002514611153, "y1998": 1.144725587086376, "y2006": 1.1173646811350333, "y2007": 1.1086324218539598, "y2004": 1.1102496406140896, "y2005": 1.11943471361418, "y2002": 1.1475230282561595, "y2003": 1.1184328424005199, "y2000": 1.1689820101690329, "y2001": 1.1721248787169682, "id": 4, "y2008": 1.0964251552643696, "y2009": 1.0776233718455337}, {"neighbors": [29, 1, 22, 7, 4], "y1995": 1.422697571371182, "y1997": 1.4427350196405593, "y1996": 1.4211843379728528, "y1999": 1.4440068434166562, "y1998": 1.4357757095632602, "y2006": 1.4405276647793266, "y2007": 1.4524121586440921, "y2004": 1.4059372049179741, "y2005": 1.4078864636665769, "y2002": 1.4197822680667809, "y2003": 1.3909220829548647, "y2000": 1.4418473669388905, "y2001": 1.4478283203013527, "id": 5, "y2008": 1.4330609762040207, "y2009": 1.4174430982377491}, {"neighbors": [12, 47, 9, 25, 20], "y1995": 1.1307388498039153, "y1997": 1.1107470843142355, "y1996": 1.1311051255854685, "y1999": 1.130881491772973, "y1998": 1.1336463608751246, "y2006": 1.1088003408832796, "y2007": 1.0840170924825394, "y2004": 1.1244623853593112, "y2005": 1.1167100811401538, "y2002": 1.1306293052597198, "y2003": 1.1194498381213465, "y2000": 1.1088813841947593, "y2001": 1.1185662918783175, "id": 6, "y2008": 1.0695920556329086, "y2009": 1.0787522517402164}, {"neighbors": [21, 1, 22, 10, 0], "y1995": 1.0470612357366649, "y1997": 1.0425337165747406, "y1996": 1.0451683097376836, "y1999": 1.0207254480945218, "y1998": 1.0323998680588111, "y2006": 1.0405109962442973, "y2007": 1.0174964540280445, "y2004": 1.0140090547678748, "y2005": 1.0317674181861733, "y2002": 0.99669586934394627, "y2003": 0.99327675611171373, "y2000": 0.99854316295509526, "y2001": 0.98802579761429143, "id": 7, "y2008": 0.9936394033949828, "y2009": 0.98279746069218921}, {"neighbors": [11, 13, 17, 18, 15], "y1995": 0.98996985668705595, "y1997": 0.99491000469481983, "y1996": 1.0014356415938011, "y1999": 1.0045584503565237, "y1998": 1.0018840754492748, "y2006": 0.92232873520447411, "y2007": 0.91284090705064902, "y2004": 0.93694786512729977, "y2005": 0.94308212820743131, "y2002": 0.96834820215592055, "y2003": 0.95335147249088092, "y2000": 0.99127006477048718, "y2001": 0.97925917470464008, "id": 8, "y2008": 0.89689832627117483, "y2009": 0.88928857608264111}, {"neighbors": [12, 6, 4, 3, 14], "y1995": 0.87418390853652306, "y1997": 0.84425695187978567, "y1996": 0.86416601430334228, "y1999": 0.83903043942542854, "y1998": 0.8404493987171674, "y2006": 0.87204140839730271, "y2007": 0.86633032299764789, "y2004": 0.86981997840756087, "y2005": 0.86837929279319737, "y2002": 0.86107306112852877, "y2003": 0.85007719735663123, "y2000": 0.85787080050645603, "y2001": 0.86036185149249467, "id": 9, "y2008": 0.84946077011565357, "y2009": 0.83287145944123797}, {"neighbors": [0, 7, 21, 23, 22], "y1995": 1.1419611801631209, "y1997": 1.1489271154554144, "y1996": 1.146602624490825, "y1999": 1.1443662376135306, "y1998": 1.1490959392942743, "y2006": 1.1049125811637337, "y2007": 1.1105984164317646, "y2004": 1.1119989015058092, "y2005": 1.1025779214946556, "y2002": 1.1259666377127024, "y2003": 1.1221399558345004, "y2000": 1.144501826035474, "y2001": 1.1234975172649961, "id": 10, "y2008": 1.1050979494645479, "y2009": 1.1002009697391872}, {"neighbors": [8, 13, 18, 17, 2], "y1995": 0.97282462974938089, "y1997": 0.96252588061647382, "y1996": 0.96700147279313231, "y1999": 0.96057686787383312, "y1998": 0.96538780087103548, "y2006": 0.91010201260822066, "y2007": 0.89280392121658247, "y2004": 0.94103988614185807, "y2005": 0.9212251863828258, "y2002": 0.94804194711420009, "y2003": 0.9543028555845573, "y2000": 0.95831051250950716, "y2001": 0.94480908623936988, "id": 11, "y2008": 0.89298242828382146, "y2009": 0.89165384824292859}, {"neighbors": [33, 9, 6, 25, 31], "y1995": 0.94325467991401402, "y1997": 0.96455242154753429, "y1996": 0.96436902092427723, "y1999": 0.94117647058823528, "y1998": 0.95243008993884537, "y2006": 0.9346681464882507, "y2007": 0.94281559150403071, "y2004": 0.96918424441756057, "y2005": 0.94781280876672958, "y2002": 0.95388717527096822, "y2003": 0.94597005193649519, "y2000": 0.94809269652332606, "y2001": 0.93539181553564288, "id": 12, "y2008": 0.965203150896216, "y2009": 0.967154410723015}, {"neighbors": [18, 17, 11, 8, 19], "y1995": 0.97478408425654373, "y1997": 0.98712808751954773, "y1996": 0.98169225257738801, "y1999": 0.985598971191053, "y1998": 0.98474769442356791, "y2006": 0.98416665248276058, "y2007": 0.98423613480079708, "y2004": 0.97399471186978948, "y2005": 0.96910087128357136, "y2002": 0.9820996926750224, "y2003": 0.98776529543110569, "y2000": 0.98687072733199255, "y2001": 0.99237486444837619, "id": 13, "y2008": 0.99823861244053191, "y2009": 0.99545704236827348}, {"neighbors": [4, 31, 3, 29, 12], "y1995": 0.85570268988941878, "y1997": 0.85986131704895119, "y1996": 0.85575915188345031, "y1999": 0.85380119644969055, "y1998": 0.85693406055397725, "y2006": 0.82803647591954255, "y2007": 0.81987360180979219, "y2004": 0.83998883284341452, "y2005": 0.83478547261894065, "y2002": 0.85472102128186755, "y2003": 0.84564834502399988, "y2000": 0.86191535266765262, "y2001": 0.84981450830432048, "id": 14, "y2008": 0.82265395167873867, "y2009": 0.83994039782937002}, {"neighbors": [19, 8, 17, 16, 13], "y1995": 0.87022046646521634, "y1997": 0.85961813213722393, "y1996": 0.85996258309339635, "y1999": 0.8394713575455558, "y1998": 0.85689572413110093, "y2006": 0.94202108334913126, "y2007": 0.94222309998743192, "y2004": 0.86763340229291142, "y2005": 0.89179316746010362, "y2002": 0.86776297543511893, "y2003": 0.86720209304280604, "y2000": 0.82785596604704892, "y2001": 0.86008789452656809, "id": 15, "y2008": 0.93902708112840494, "y2009": 0.94479183757120588}, {"neighbors": [28, 26, 15, 19, 32], "y1995": 0.90134907329491731, "y1997": 0.90403990934606904, "y1996": 0.904077381347274, "y1999": 0.90399237579083946, "y1998": 0.90201769385650832, "y2006": 0.91108803862404764, "y2007": 0.90543476309316473, "y2004": 0.94338264626469681, "y2005": 0.91981795862151561, "y2002": 0.93695966482853577, "y2003": 0.94242697007039, "y2000": 0.90906631602055099, "y2001": 0.92693339421265908, "id": 16, "y2008": 0.91737137682250491, "y2009": 0.94793657442067902}, {"neighbors": [13, 18, 11, 19, 8], "y1995": 1.1977611005602815, "y1997": 1.1843915817489725, "y1996": 1.1822256425225894, "y1999": 1.1928672308275252, "y1998": 1.1826786457339149, "y2006": 1.2392938410349985, "y2007": 1.2341867605077472, "y2004": 1.2385704217423759, "y2005": 1.2441989281116201, "y2002": 1.2262477774195681, "y2003": 1.2239707531714479, "y2000": 1.2017286912636342, "y2001": 1.2132869128474402, "id": 17, "y2008": 1.2362673914436095, "y2009": 1.2675439750795283}, {"neighbors": [13, 17, 11, 8, 19], "y1995": 1.2491967813733067, "y1997": 1.2699116090397236, "y1996": 1.2575477330927329, "y1999": 1.3062566740535762, "y1998": 1.2802065055312271, "y2006": 1.3210776560048689, "y2007": 1.329362443219563, "y2004": 1.3054484140490119, "y2005": 1.3030330249408666, "y2002": 1.3257518058685978, "y2003": 1.3079549159235695, "y2000": 1.3479002255103918, "y2001": 1.3439986302151703, "id": 18, "y2008": 1.3300124123891741, "y2009": 1.3328846185074705}, {"neighbors": [26, 17, 28, 15, 16], "y1995": 1.0676800411188558, "y1997": 1.0363730321443168, "y1996": 1.0379927554499979, "y1999": 1.0329609259280523, "y1998": 1.027684488045026, "y2006": 0.94241549375546196, "y2007": 0.92754546923532677, "y2004": 0.99614160423102482, "y2005": 0.97356208269708677, "y2002": 1.0274762326434594, "y2003": 1.0316273366809443, "y2000": 1.0505901631347052, "y2001": 1.0340505678899605, "id": 19, "y2008": 0.92549226593721745, "y2009": 0.92138101880290568}, {"neighbors": [30, 25, 24, 37, 47], "y1995": 1.0947561397632881, "y1997": 1.1165429913770684, "y1996": 1.1152679554712275, "y1999": 1.1314326394231322, "y1998": 1.1310394841195361, "y2006": 1.1090538904302065, "y2007": 1.1057776900012568, "y2004": 1.1402994437897009, "y2005": 1.1197940058085571, "y2002": 1.133670175399079, "y2003": 1.139822558851451, "y2000": 1.1388962186541665, "y2001": 1.1244221220249986, "id": 20, "y2008": 1.1116682481010467, "y2009": 1.0998515545336902}, {"neighbors": [23, 22, 7, 10, 34], "y1995": 0.76530058421804126, "y1997": 0.76542450966153397, "y1996": 0.76612841163904621, "y1999": 0.76014283909933289, "y1998": 0.7672268310234307, "y2006": 0.76842416021983684, "y2007": 0.77487117798086069, "y2004": 0.76533287692895391, "y2005": 0.78205934309410463, "y2002": 0.76156903267949927, "y2003": 0.76651951668098528, "y2000": 0.74480073263159763, "y2001": 0.76098396210261965, "id": 21, "y2008": 0.77768682781054099, "y2009": 0.78801192267396702}, {"neighbors": [21, 34, 5, 7, 29], "y1995": 0.98391336093764348, "y1997": 0.98295341320156315, "y1996": 0.98075815675295552, "y1999": 0.96913802803963667, "y1998": 0.97386015032669815, "y2006": 0.93965462091114671, "y2007": 0.93069644684632924, "y2004": 0.9635616201227476, "y2005": 0.94745351657235244, "y2002": 0.97209860866113018, "y2003": 0.97441312580606143, "y2000": 0.97370819354423843, "y2001": 0.96419154157867693, "id": 22, "y2008": 0.94020973488297466, "y2009": 0.94358232339833159}, {"neighbors": [21, 10, 22, 34, 7], "y1995": 0.83561828119099946, "y1997": 0.81738501913392403, "y1996": 0.82298088022609361, "y1999": 0.80904800725677739, "y1998": 0.81748588141426259, "y2006": 0.87170334233473346, "y2007": 0.8786379876833581, "y2004": 0.85954307066870839, "y2005": 0.86790023653402792, "y2002": 0.83451612857812574, "y2003": 0.85175031934895873, "y2000": 0.80071489233375537, "y2001": 0.83358255807316928, "id": 23, "y2008": 0.87497981001981484, "y2009": 0.87888675419592222}, {"neighbors": [27, 20, 30, 32, 47], "y1995": 0.98845573274970278, "y1997": 0.99665282989553183, "y1996": 1.0209242772035507, "y1999": 0.99386618594343845, "y1998": 0.99141823200404444, "y2006": 0.97906748937234156, "y2007": 0.9932312332800689, "y2004": 1.0111665058188304, "y2005": 0.9998802359352077, "y2002": 0.99669586934394627, "y2003": 1.0255909749831356, "y2000": 0.98733194819247994, "y2001": 0.99644997431653437, "id": 24, "y2008": 1.0020493856497013, "y2009": 0.99602148231561483}, {"neighbors": [20, 33, 6, 30, 12], "y1995": 1.1493091345649815, "y1997": 1.143009615936718, "y1996": 1.1524194939429724, "y1999": 1.1398468268822266, "y1998": 1.1426554202510555, "y2006": 1.0889107875354573, "y2007": 1.0860369499254896, "y2004": 1.0856975145267398, "y2005": 1.1244348633192611, "y2002": 1.0423089214343333, "y2003": 1.0557727834721793, "y2000": 1.0831239730629278, "y2001": 1.0519262599166714, "id": 25, "y2008": 1.0599731384290745, "y2009": 1.0216094265950888}, {"neighbors": [28, 19, 16, 32, 17], "y1995": 1.1136826889802023, "y1997": 1.1189343096757198, "y1996": 1.1057147027213501, "y1999": 1.1432271991365353, "y1998": 1.1377866945457653, "y2006": 1.1268023587150906, "y2007": 1.1235793669317915, "y2004": 1.1482023546040769, "y2005": 1.1238659840114973, "y2002": 1.1600919581655105, "y2003": 1.1446778932605579, "y2000": 1.1825702862895446, "y2001": 1.1622624279436105, "id": 26, "y2008": 1.115925801617498, "y2009": 1.1257082797404696}, {"neighbors": [32, 24, 36, 16, 28], "y1995": 1.303794309231981, "y1997": 1.3120636604057812, "y1996": 1.3075218596998686, "y1999": 1.3062566740535762, "y1998": 1.3153226688859194, "y2006": 1.2865667454509278, "y2007": 1.2973409698906584, "y2004": 1.2683078569016086, "y2005": 1.2617743046198988, "y2002": 1.2920319347677043, "y2003": 1.2718351646774422, "y2000": 1.3121023910310281, "y2001": 1.2998915587009874, "id": 27, "y2008": 1.2939020510829768, "y2009": 1.2934544564717687}, {"neighbors": [26, 16, 19, 32, 27], "y1995": 0.83953719020532513, "y1997": 0.82006005316292385, "y1996": 0.82701447583159737, "y1999": 0.80294863992835086, "y1998": 0.8118887636743225, "y2006": 0.8389109342655191, "y2007": 0.84349246817602375, "y2004": 0.83108634437662732, "y2005": 0.84373783646216949, "y2002": 0.82596790474192727, "y2003": 0.82435704751379402, "y2000": 0.78772975118465016, "y2001": 0.82848010958278628, "id": 28, "y2008": 0.85637272428125033, "y2009": 0.86539395164519117}, {"neighbors": [5, 39, 22, 14, 31], "y1995": 1.2345008725695852, "y1997": 1.2353793515744536, "y1996": 1.2426021999018138, "y1999": 1.2452262575926329, "y1998": 1.2358129278404693, "y2006": 1.2365329681906834, "y2007": 1.2796200872578414, "y2004": 1.1967443443492951, "y2005": 1.2153657295128597, "y2002": 1.1937780418204111, "y2003": 1.1835533748469893, "y2000": 1.2256766974812463, "y2001": 1.2112664802237314, "id": 29, "y2008": 1.2796839248335934, "y2009": 1.2590773758694083}, {"neighbors": [37, 20, 24, 25, 27], "y1995": 0.97696620404861145, "y1997": 0.98035944080980575, "y1996": 0.9740071914763756, "y1999": 0.95543282313901556, "y1998": 0.97581530789338955, "y2006": 0.92100464312607799, "y2007": 0.9147530387633086, "y2004": 0.9298883479571457, "y2005": 0.93442917452618346, "y2002": 0.93679072759857129, "y2003": 0.92540049332494034, "y2000": 0.96480308308405971, "y2001": 0.9468637634838194, "id": 30, "y2008": 0.90249622070947177, "y2009": 0.90213630440783921}, {"neighbors": [35, 14, 33, 12, 4], "y1995": 0.84986885942491119, "y1997": 0.84295996568390696, "y1996": 0.89868510090623221, "y1999": 0.85659367787716301, "y1998": 0.87280533962476625, "y2006": 0.92562487931452408, "y2007": 0.96635366357254426, "y2004": 0.92698332540482575, "y2005": 0.94745351657235244, "y2002": 0.90448992922937876, "y2003": 0.95495898185605821, "y2000": 0.88937573313051443, "y2001": 0.89440100450887505, "id": 31, "y2008": 1.025203118044723, "y2009": 1.0394296020754366}, {"neighbors": [36, 27, 28, 16, 26], "y1995": 1.0192280751235561, "y1997": 1.0097442843101825, "y1996": 1.0025820319237864, "y1999": 0.99765073314119712, "y1998": 1.0030341681355639, "y2006": 0.94779637858468868, "y2007": 0.93759089358493275, "y2004": 0.97583768316642261, "y2005": 0.96101679691008712, "y2002": 0.99747298060178258, "y2003": 0.99550758543481688, "y2000": 1.0075901875261932, "y2001": 0.99192968437874551, "id": 32, "y2008": 0.93353431146829191, "y2009": 0.94121705123804411}, {"neighbors": [44, 25, 12, 35, 31], "y1995": 0.86367410708901315, "y1997": 0.85544345781923936, "y1996": 0.85558931627900803, "y1999": 0.84336613427334628, "y1998": 0.85103025143102673, "y2006": 0.89455097373003656, "y2007": 0.88283929116469462, "y2004": 0.85951183386707053, "y2005": 0.87194227372077004, "y2002": 0.84667960913556228, "y2003": 0.84374557883664714, "y2000": 0.83434853662160158, "y2001": 0.85813595114434105, "id": 33, "y2008": 0.90349490610221961, "y2009": 0.9060067497610369}, {"neighbors": [22, 39, 21, 29, 23], "y1995": 1.0094753356447226, "y1997": 1.0069881886439402, "y1996": 1.0041105523637666, "y1999": 0.99291086334982948, "y1998": 0.99513686502304577, "y2006": 0.96382634438484593, "y2007": 0.95011400973122428, "y2004": 0.975119236728752, "y2005": 0.96134614808826613, "y2002": 0.99291167539274383, "y2003": 0.98983209318633369, "y2000": 1.0058162611397035, "y2001": 0.98850522230466298, "id": 34, "y2008": 0.94346860300667812, "y2009": 0.9463776450423077}, {"neighbors": [31, 38, 44, 33, 14], "y1995": 1.0571257066143651, "y1997": 1.0575301194645879, "y1996": 1.0545941857842291, "y1999": 1.0510385688532684, "y1998": 1.0488078570498685, "y2006": 1.0247627521629479, "y2007": 1.0234752320591773, "y2004": 1.0329697933620496, "y2005": 1.0219168238570018, "y2002": 1.0420048344203974, "y2003": 1.0402553971511816, "y2000": 1.0480002306104303, "y2001": 1.030249414987729, "id": 35, "y2008": 1.0251768368501768, "y2009": 1.0435957064486703}, {"neighbors": [32, 43, 27, 28, 42], "y1995": 1.070841888164505, "y1997": 1.0793762307014196, "y1996": 1.0666949726007404, "y1999": 1.0794043012481198, "y1998": 1.0738798776109699, "y2006": 1.087727556316465, "y2007": 1.0885954360198933, "y2004": 1.1032213602455734, "y2005": 1.0916793915985508, "y2002": 1.0938347765734742, "y2003": 1.1052447043433509, "y2000": 1.0531800956589803, "y2001": 1.0745277096056161, "id": 36, "y2008": 1.0917733838297285, "y2009": 1.1096083021948762}, {"neighbors": [30, 40, 20, 42, 41], "y1995": 0.8671922185905101, "y1997": 0.86675155621455668, "y1996": 0.86628895935887062, "y1999": 0.86511809486628932, "y1998": 0.86425631732335095, "y2006": 0.84488343470424199, "y2007": 0.83374328958471722, "y2004": 0.84517414191529749, "y2005": 0.84843857600526962, "y2002": 0.85411284725399572, "y2003": 0.84886336375435456, "y2000": 0.86287327291635718, "y2001": 0.8516979624450659, "id": 37, "y2008": 0.82812044014430564, "y2009": 0.82878598934619596}, {"neighbors": [35, 31, 45, 39, 44], "y1995": 0.8838921149583755, "y1997": 0.90282398478743275, "y1996": 0.92288667453925455, "y1999": 0.92023285988219217, "y1998": 0.91229185518735723, "y2006": 0.93869676706720051, "y2007": 0.96947770975097391, "y2004": 0.99223700402629367, "y2005": 0.97984969609868555, "y2002": 0.93682451504456421, "y2003": 0.98655146182882891, "y2000": 0.92652175166361039, "y2001": 0.94278865361566122, "id": 38, "y2008": 1.0036262573224608, "y2009": 0.98102350657197357}, {"neighbors": [29, 34, 38, 22, 35], "y1995": 0.970820642185237, "y1997": 0.94534081352108112, "y1996": 0.95320232993219844, "y1999": 0.93967000034446724, "y1998": 0.94215592860799646, "y2006": 0.91035556215514757, "y2007": 0.90430364292511256, "y2004": 0.92879505989982103, "y2005": 0.9211054223180335, "y2002": 0.93412151936513388, "y2003": 0.93501274320242933, "y2000": 0.93092108910210503, "y2001": 0.92662519262599163, "id": 39, "y2008": 0.89994694483851023, "y2009": 0.9007386435858511}, {"neighbors": [41, 37, 42, 30, 45], "y1995": 0.95861858457245008, "y1997": 0.98254810501535106, "y1996": 0.95774543235102894, "y1999": 0.98684823919808018, "y1998": 0.98919471947721893, "y2006": 0.97163003599581876, "y2007": 0.97007020126757271, "y2004": 0.9493488753775261, "y2005": 0.97152609359561659, "y2002": 0.95601578436851964, "y2003": 0.94905384541254967, "y2000": 0.98882204635713133, "y2001": 0.97662233890759653, "id": 40, "y2008": 0.97158948117089283, "y2009": 0.95884908006927827}, {"neighbors": [40, 45, 44, 37, 42], "y1995": 0.83980438854721107, "y1997": 0.85746999875029983, "y1996": 0.84726737166133714, "y1999": 0.85567509846023126, "y1998": 0.85467221160427542, "y2006": 0.8333891885768886, "y2007": 0.83511679264592342, "y2004": 0.81743586206088703, "y2005": 0.83550405700769481, "y2002": 0.84502402428191115, "y2003": 0.82645665158259707, "y2000": 0.84818516243622177, "y2001": 0.85265681182580899, "id": 41, "y2008": 0.82136617314598481, "y2009": 0.80921873783836296}, {"neighbors": [43, 40, 46, 37, 36], "y1995": 0.95118156405662746, "y1997": 0.94688098462868708, "y1996": 0.9466212002600608, "y1999": 0.95124410099780687, "y1998": 0.95085829660091703, "y2006": 0.96895367966714574, "y2007": 0.9700163384024274, "y2004": 0.97583768316642261, "y2005": 0.95571723704302525, "y2002": 0.96804411514198463, "y2003": 0.97136213864358201, "y2000": 0.95440787445922959, "y2001": 0.96364362764682376, "id": 42, "y2008": 0.97082732652905901, "y2009": 0.9878236640328002}, {"neighbors": [36, 42, 32, 27, 46], "y1995": 1.0891004415267045, "y1997": 1.0849289528525252, "y1996": 1.0824896838138709, "y1999": 1.0945424900391545, "y1998": 1.0865692335830259, "y2006": 1.1450297539219478, "y2007": 1.1447474729339102, "y2004": 1.1334273474293739, "y2005": 1.1468606844516303, "y2002": 1.1229257675733433, "y2003": 1.1302103089739621, "y2000": 1.1055818811158884, "y2001": 1.1214085953998059, "id": 43, "y2008": 1.1408403740471014, "y2009": 1.1614292649793569}, {"neighbors": [33, 41, 45, 35, 40], "y1995": 1.0633603345917013, "y1997": 1.0869149629649646, "y1996": 1.0736582323828732, "y1999": 1.1166986255755473, "y1998": 1.0976484597942771, "y2006": 1.0839806574563229, "y2007": 1.0983176831786272, "y2004": 1.0927882684985315, "y2005": 1.0700320368873319, "y2002": 1.0881584856466706, "y2003": 1.0804431312806149, "y2000": 1.1185670222649935, "y2001": 1.0976428286056732, "id": 44, "y2008": 1.0929823187788443, "y2009": 1.0917612486217978}, {"neighbors": [41, 44, 40, 35, 33], "y1995": 0.79772064970019041, "y1997": 0.7858115114280021, "y1996": 0.78829195801876151, "y1999": 0.77035744221561353, "y1998": 0.77615921755360906, "y2006": 0.79949806580432425, "y2007": 0.80172181625581262, "y2004": 0.79603865293896003, "y2005": 0.78966436120841943, "y2002": 0.81437881076636964, "y2003": 0.80788827809912023, "y2000": 0.77751193519846906, "y2001": 0.79902973574567659, "id": 45, "y2008": 0.82168154748053679, "y2009": 0.85587910681858015}, {"neighbors": [42, 43, 40, 36, 37], "y1995": 1.0052446952315301, "y1997": 1.0047589936197736, "y1996": 1.0000769567582628, "y1999": 1.0063956091903872, "y1998": 1.0061394183885444, "y2006": 0.97292595590233411, "y2007": 0.96519561197191939, "y2004": 0.99030032232474696, "y2005": 0.97682565346267858, "y2002": 1.0081498135355325, "y2003": 1.0057431552702318, "y2000": 1.0016297948675874, "y2001": 0.99860738542320637, "id": 46, "y2008": 0.9617340332161447, "y2009": 0.95890283625473927}, {"neighbors": [20, 6, 24, 25, 30], "y1995": 0.95808418788867844, "y1997": 0.9654440995572009, "y1996": 0.93825679674127938, "y1999": 0.96987289157318213, "y1998": 0.95561201303757848, "y2006": 1.1704973973021624, "y2007": 1.1702515395802287, "y2004": 1.0533361880299275, "y2005": 1.0983262971945267, "y2002": 1.0078119390756035, "y2003": 1.0348423554112989, "y2000": 0.96608031008233231, "y2001": 0.99727184521431422, "id": 47, "y2008": 1.1873055260044207, "y2009": 1.1424264534188653}] diff --git a/release/python/0.1.0/crankshaft/test/helper.py b/release/python/0.1.0/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.1.0/crankshaft/test/mock_plpy.py b/release/python/0.1.0/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..a982ebe --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/mock_plpy.py @@ -0,0 +1,52 @@ +import re + +class MockCursor: + def __init__(self, data): + self.cursor_pos = 0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos : self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def debug(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def cursor(self, query): + data = self.execute(query) + return MockCursor(data) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.1.0/crankshaft/test/test_cluster_kmeans.py b/release/python/0.1.0/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.1.0/crankshaft/test/test_clustering_moran.py b/release/python/0.1.0/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..2b683cf --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,88 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.params_markov = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.1.0/crankshaft/test/test_pysal_utils.py b/release/python/0.1.0/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..171fdbc --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,142 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + self.params_array = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " + + ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params_array), ans_array) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"andy\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" <> 0" + + ans_array = "idx_replace.\"_2013_dec\" IS NOT NULL AND " \ + "idx_replace.\"_2014_jan\" IS NOT NULL AND " \ + "idx_replace.\"_2014_feb\" IS NOT NULL" + + self.assertEqual(pu.query_attr_where(self.params), ans) + self.assertEqual(pu.query_attr_where(self.params_array), ans_array) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + ans_array = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"_2013_dec\" IS NOT NULL AND " \ + "j.\"_2014_jan\" IS NOT NULL AND " \ + "j.\"_2014_feb\" IS NOT NULL " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"_2013_dec\" IS NOT NULL AND " \ + "i.\"_2014_jan\" IS NOT NULL AND " \ + "i.\"_2014_feb\" IS NOT NULL "\ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + self.assertEqual(pu.knn(self.params_array), ans_array) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/release/python/0.1.0/crankshaft/test/test_segmentation.py b/release/python/0.1.0/crankshaft/test/test_segmentation.py new file mode 100644 index 0000000..d02e8b1 --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/test_segmentation.py @@ -0,0 +1,64 @@ +import unittest +import numpy as np +from helper import plpy, fixture_file +import crankshaft.segmentation as segmentation +import json + +class SegmentationTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + + def generate_random_data(self,n_samples,random_state, row_type=False): + x1 = random_state.uniform(size=n_samples) + x2 = random_state.uniform(size=n_samples) + x3 = random_state.randint(0, 4, size=n_samples) + + y = x1+x2*x2+x3 + cartodb_id = range(len(x1)) + + if row_type: + return [ {'features': vals} for vals in zip(x1,x2,x3)], y + else: + return [dict( zip(['x1','x2','x3','target', 'cartodb_id'],[x1,x2,x3,y,cartodb_id]))] + + def test_replace_nan_with_mean(self): + test_array = np.array([1.2, np.nan, 3.2, np.nan, np.nan]) + + def test_create_and_predict_segment(self): + n_samples = 1000 + + random_state_train = np.random.RandomState(13) + random_state_test = np.random.RandomState(134) + training_data = self.generate_random_data(n_samples, random_state_train) + test_data, test_y = self.generate_random_data(n_samples, random_state_test, row_type=True) + + + ids = [{'cartodb_ids': range(len(test_data))}] + rows = [{'x1': 0,'x2':0,'x3':0,'y':0,'cartodb_id':0}] + + plpy._define_result('select \* from \(select \* from training\) a limit 1',rows) + plpy._define_result('.*from \(select \* from training\) as a' ,training_data) + plpy._define_result('select array_agg\(cartodb\_id order by cartodb\_id\) as cartodb_ids from \(.*\) a',ids) + plpy._define_result('.*select \* from test.*' ,test_data) + + model_parameters = {'n_estimators': 1200, + 'max_depth': 3, + 'subsample' : 0.5, + 'learning_rate': 0.01, + 'min_samples_leaf': 1} + + result = segmentation.create_and_predict_segment( + 'select * from training', + 'target', + 'select * from test', + model_parameters) + + prediction = [r[1] for r in result] + + accuracy =np.sqrt(np.mean( np.square( np.array(prediction) - np.array(test_y)))) + + self.assertEqual(len(result),len(test_data)) + self.assertTrue( result[0][2] < 0.01) + self.assertTrue( accuracy < 0.5*np.mean(test_y) ) diff --git a/release/python/0.1.0/crankshaft/test/test_space_time_dynamics.py b/release/python/0.1.0/crankshaft/test/test_space_time_dynamics.py new file mode 100644 index 0000000..54ffc9d --- /dev/null +++ b/release/python/0.1.0/crankshaft/test/test_space_time_dynamics.py @@ -0,0 +1,324 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.space_time_dynamics as std +from crankshaft import random_seeds +import json + +class SpaceTimeTests(unittest.TestCase): + """Testing class for Markov Functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors_markov.json')).read()) + self.markov_data = json.loads(open(fixture_file('markov.json')).read()) + + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T + + self.transition_matrix = np.array([ + [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], + [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], + [ 0. , 0.14 , 0.74 , 0.12 , 0. ], + [ 0. , 0.03571429, 0.32142857, 0.57142857, 0.07142857], + [ 0. , 0. , 0. , 0.16666667, 0.83333333]], + [[ 0.79831933, 0.16806723, 0.03361345, 0. , 0. ], + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0.00537634, 0.06989247, 0.8655914 , 0.05913978, 0. ], + [ 0. , 0. , 0.06372549, 0.90196078, 0.03431373], + [ 0. , 0. , 0. , 0.19444444, 0.80555556]], + [[ 0.84693878, 0.15306122, 0. , 0. , 0. ], + [ 0.08133971, 0.78947368, 0.1291866 , 0. , 0. ], + [ 0.00518135, 0.0984456 , 0.79274611, 0.0984456 , 0.00518135], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0. , 0. , 0. , 0.10204082, 0.89795918]], + [[ 0.8852459 , 0.09836066, 0. , 0.01639344, 0. ], + [ 0.03875969, 0.81395349, 0.13953488, 0. , 0.00775194], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0.02339181, 0.12865497, 0.75438596, 0.09356725], + [ 0. , 0. , 0. , 0.09661836, 0.90338164]], + [[ 0.33333333, 0.66666667, 0. , 0. , 0. ], + [ 0.0483871 , 0.77419355, 0.16129032, 0.01612903, 0. ], + [ 0.01149425, 0.16091954, 0.74712644, 0.08045977, 0. ], + [ 0. , 0.01036269, 0.06217617, 0.89637306, 0.03108808], + [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] + ) + + def test_spatial_markov(self): + """Test Spatial Markov.""" + data = [ { 'id': d['id'], + 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'], + 'neighbors': d['neighbors'] } for d in self.neighbors_data] + print(str(data[0])) + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + + result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id') + + self.assertTrue(result != None) + result = [(row[0], row[1], row[2], row[3], row[4]) for row in result] + print result[0] + expected = self.markov_data + for ([res_trend, res_up, res_down, res_vol, res_id], + [exp_trend, exp_up, exp_down, exp_vol, exp_id] + ) in zip(result, expected): + self.assertAlmostEqual(res_trend, exp_trend) + + def test_get_time_data(self): + """Test get_time_data""" + data = [ { 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'] } for d in self.neighbors_data] + + result = std.get_time_data(data, ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']) + + ## expected was prepared from PySAL example: + ### f = ps.open(ps.examples.get_path("usjoin.csv")) + ### pci = np.array([f.by_col[str(y)] for y in range(1995, 2010)]).transpose() + ### rpci = pci / (pci.mean(axis = 0)) + + expected = np.array([[ 0.87654416, 0.863147, 0.85637567, 0.84811668, 0.8446154, 0.83271652 + , 0.83786314, 0.85012593, 0.85509656, 0.86416612, 0.87119375, 0.86302631 + , 0.86148267, 0.86252252, 0.86746356], + [ 0.9188951, 0.91757931, 0.92333258, 0.92517289, 0.92552388, 0.90746978 + , 0.89830489, 0.89431991, 0.88924794, 0.89815176, 0.91832091, 0.91706054 + , 0.90139505, 0.87897455, 0.86216858], + [ 0.82591007, 0.82548596, 0.81989793, 0.81503235, 0.81731522, 0.78964559 + , 0.80584442, 0.8084998, 0.82258551, 0.82668196, 0.82373724, 0.81814804 + , 0.83675961, 0.83574199, 0.84647177], + [ 1.09088176, 1.08537689, 1.08456418, 1.08415404, 1.09898841, 1.14506948 + , 1.12151133, 1.11160697, 1.10888621, 1.11399806, 1.12168029, 1.13164797 + , 1.12958508, 1.11371818, 1.09936775], + [ 1.10731446, 1.11373944, 1.13283638, 1.14472559, 1.15910025, 1.16898201 + , 1.17212488, 1.14752303, 1.11843284, 1.11024964, 1.11943471, 1.11736468 + , 1.10863242, 1.09642516, 1.07762337], + [ 1.42269757, 1.42118434, 1.44273502, 1.43577571, 1.44400684, 1.44184737 + , 1.44782832, 1.41978227, 1.39092208, 1.4059372, 1.40788646, 1.44052766 + , 1.45241216, 1.43306098, 1.4174431 ], + [ 1.13073885, 1.13110513, 1.11074708, 1.13364636, 1.13088149, 1.10888138 + , 1.11856629, 1.13062931, 1.11944984, 1.12446239, 1.11671008, 1.10880034 + , 1.08401709, 1.06959206, 1.07875225], + [ 1.04706124, 1.04516831, 1.04253372, 1.03239987, 1.02072545, 0.99854316 + , 0.9880258, 0.99669587, 0.99327676, 1.01400905, 1.03176742, 1.040511 + , 1.01749645, 0.9936394, 0.98279746], + [ 0.98996986, 1.00143564, 0.99491, 1.00188408, 1.00455845, 0.99127006 + , 0.97925917, 0.9683482, 0.95335147, 0.93694787, 0.94308213, 0.92232874 + , 0.91284091, 0.89689833, 0.88928858], + [ 0.87418391, 0.86416601, 0.84425695, 0.8404494, 0.83903044, 0.8578708 + , 0.86036185, 0.86107306, 0.8500772, 0.86981998, 0.86837929, 0.87204141 + , 0.86633032, 0.84946077, 0.83287146], + [ 1.14196118, 1.14660262, 1.14892712, 1.14909594, 1.14436624, 1.14450183 + , 1.12349752, 1.12596664, 1.12213996, 1.1119989, 1.10257792, 1.10491258 + , 1.11059842, 1.10509795, 1.10020097], + [ 0.97282463, 0.96700147, 0.96252588, 0.9653878, 0.96057687, 0.95831051 + , 0.94480909, 0.94804195, 0.95430286, 0.94103989, 0.92122519, 0.91010201 + , 0.89280392, 0.89298243, 0.89165385], + [ 0.94325468, 0.96436902, 0.96455242, 0.95243009, 0.94117647, 0.9480927 + , 0.93539182, 0.95388718, 0.94597005, 0.96918424, 0.94781281, 0.93466815 + , 0.94281559, 0.96520315, 0.96715441], + [ 0.97478408, 0.98169225, 0.98712809, 0.98474769, 0.98559897, 0.98687073 + , 0.99237486, 0.98209969, 0.9877653, 0.97399471, 0.96910087, 0.98416665 + , 0.98423613, 0.99823861, 0.99545704], + [ 0.85570269, 0.85575915, 0.85986132, 0.85693406, 0.8538012, 0.86191535 + , 0.84981451, 0.85472102, 0.84564835, 0.83998883, 0.83478547, 0.82803648 + , 0.8198736, 0.82265395, 0.8399404 ], + [ 0.87022047, 0.85996258, 0.85961813, 0.85689572, 0.83947136, 0.82785597 + , 0.86008789, 0.86776298, 0.86720209, 0.8676334, 0.89179317, 0.94202108 + , 0.9422231, 0.93902708, 0.94479184], + [ 0.90134907, 0.90407738, 0.90403991, 0.90201769, 0.90399238, 0.90906632 + , 0.92693339, 0.93695966, 0.94242697, 0.94338265, 0.91981796, 0.91108804 + , 0.90543476, 0.91737138, 0.94793657], + [ 1.1977611, 1.18222564, 1.18439158, 1.18267865, 1.19286723, 1.20172869 + , 1.21328691, 1.22624778, 1.22397075, 1.23857042, 1.24419893, 1.23929384 + , 1.23418676, 1.23626739, 1.26754398], + [ 1.24919678, 1.25754773, 1.26991161, 1.28020651, 1.30625667, 1.34790023 + , 1.34399863, 1.32575181, 1.30795492, 1.30544841, 1.30303302, 1.32107766 + , 1.32936244, 1.33001241, 1.33288462], + [ 1.06768004, 1.03799276, 1.03637303, 1.02768449, 1.03296093, 1.05059016 + , 1.03405057, 1.02747623, 1.03162734, 0.9961416, 0.97356208, 0.94241549 + , 0.92754547, 0.92549227, 0.92138102], + [ 1.09475614, 1.11526796, 1.11654299, 1.13103948, 1.13143264, 1.13889622 + , 1.12442212, 1.13367018, 1.13982256, 1.14029944, 1.11979401, 1.10905389 + , 1.10577769, 1.11166825, 1.09985155], + [ 0.76530058, 0.76612841, 0.76542451, 0.76722683, 0.76014284, 0.74480073 + , 0.76098396, 0.76156903, 0.76651952, 0.76533288, 0.78205934, 0.76842416 + , 0.77487118, 0.77768683, 0.78801192], + [ 0.98391336, 0.98075816, 0.98295341, 0.97386015, 0.96913803, 0.97370819 + , 0.96419154, 0.97209861, 0.97441313, 0.96356162, 0.94745352, 0.93965462 + , 0.93069645, 0.94020973, 0.94358232], + [ 0.83561828, 0.82298088, 0.81738502, 0.81748588, 0.80904801, 0.80071489 + , 0.83358256, 0.83451613, 0.85175032, 0.85954307, 0.86790024, 0.87170334 + , 0.87863799, 0.87497981, 0.87888675], + [ 0.98845573, 1.02092428, 0.99665283, 0.99141823, 0.99386619, 0.98733195 + , 0.99644997, 0.99669587, 1.02559097, 1.01116651, 0.99988024, 0.97906749 + , 0.99323123, 1.00204939, 0.99602148], + [ 1.14930913, 1.15241949, 1.14300962, 1.14265542, 1.13984683, 1.08312397 + , 1.05192626, 1.04230892, 1.05577278, 1.08569751, 1.12443486, 1.08891079 + , 1.08603695, 1.05997314, 1.02160943], + [ 1.11368269, 1.1057147, 1.11893431, 1.13778669, 1.1432272, 1.18257029 + , 1.16226243, 1.16009196, 1.14467789, 1.14820235, 1.12386598, 1.12680236 + , 1.12357937, 1.1159258, 1.12570828], + [ 1.30379431, 1.30752186, 1.31206366, 1.31532267, 1.30625667, 1.31210239 + , 1.29989156, 1.29203193, 1.27183516, 1.26830786, 1.2617743, 1.28656675 + , 1.29734097, 1.29390205, 1.29345446], + [ 0.83953719, 0.82701448, 0.82006005, 0.81188876, 0.80294864, 0.78772975 + , 0.82848011, 0.8259679, 0.82435705, 0.83108634, 0.84373784, 0.83891093 + , 0.84349247, 0.85637272, 0.86539395], + [ 1.23450087, 1.2426022, 1.23537935, 1.23581293, 1.24522626, 1.2256767 + , 1.21126648, 1.19377804, 1.18355337, 1.19674434, 1.21536573, 1.23653297 + , 1.27962009, 1.27968392, 1.25907738], + [ 0.9769662, 0.97400719, 0.98035944, 0.97581531, 0.95543282, 0.96480308 + , 0.94686376, 0.93679073, 0.92540049, 0.92988835, 0.93442917, 0.92100464 + , 0.91475304, 0.90249622, 0.9021363 ], + [ 0.84986886, 0.8986851, 0.84295997, 0.87280534, 0.85659368, 0.88937573 + , 0.894401, 0.90448993, 0.95495898, 0.92698333, 0.94745352, 0.92562488 + , 0.96635366, 1.02520312, 1.0394296 ], + [ 1.01922808, 1.00258203, 1.00974428, 1.00303417, 0.99765073, 1.00759019 + , 0.99192968, 0.99747298, 0.99550759, 0.97583768, 0.9610168, 0.94779638 + , 0.93759089, 0.93353431, 0.94121705], + [ 0.86367411, 0.85558932, 0.85544346, 0.85103025, 0.84336613, 0.83434854 + , 0.85813595, 0.84667961, 0.84374558, 0.85951183, 0.87194227, 0.89455097 + , 0.88283929, 0.90349491, 0.90600675], + [ 1.00947534, 1.00411055, 1.00698819, 0.99513687, 0.99291086, 1.00581626 + , 0.98850522, 0.99291168, 0.98983209, 0.97511924, 0.96134615, 0.96382634 + , 0.95011401, 0.9434686, 0.94637765], + [ 1.05712571, 1.05459419, 1.05753012, 1.04880786, 1.05103857, 1.04800023 + , 1.03024941, 1.04200483, 1.0402554, 1.03296979, 1.02191682, 1.02476275 + , 1.02347523, 1.02517684, 1.04359571], + [ 1.07084189, 1.06669497, 1.07937623, 1.07387988, 1.0794043, 1.0531801 + , 1.07452771, 1.09383478, 1.1052447, 1.10322136, 1.09167939, 1.08772756 + , 1.08859544, 1.09177338, 1.1096083 ], + [ 0.86719222, 0.86628896, 0.86675156, 0.86425632, 0.86511809, 0.86287327 + , 0.85169796, 0.85411285, 0.84886336, 0.84517414, 0.84843858, 0.84488343 + , 0.83374329, 0.82812044, 0.82878599], + [ 0.88389211, 0.92288667, 0.90282398, 0.91229186, 0.92023286, 0.92652175 + , 0.94278865, 0.93682452, 0.98655146, 0.992237, 0.9798497, 0.93869677 + , 0.96947771, 1.00362626, 0.98102351], + [ 0.97082064, 0.95320233, 0.94534081, 0.94215593, 0.93967, 0.93092109 + , 0.92662519, 0.93412152, 0.93501274, 0.92879506, 0.92110542, 0.91035556 + , 0.90430364, 0.89994694, 0.90073864], + [ 0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824, 0.98882205 + , 0.97662234, 0.95601578, 0.94905385, 0.94934888, 0.97152609, 0.97163004 + , 0.9700702, 0.97158948, 0.95884908], + [ 0.83980439, 0.84726737, 0.85747, 0.85467221, 0.8556751, 0.84818516 + , 0.85265681, 0.84502402, 0.82645665, 0.81743586, 0.83550406, 0.83338919 + , 0.83511679, 0.82136617, 0.80921874], + [ 0.95118156, 0.9466212, 0.94688098, 0.9508583, 0.9512441, 0.95440787 + , 0.96364363, 0.96804412, 0.97136214, 0.97583768, 0.95571724, 0.96895368 + , 0.97001634, 0.97082733, 0.98782366], + [ 1.08910044, 1.08248968, 1.08492895, 1.08656923, 1.09454249, 1.10558188 + , 1.1214086, 1.12292577, 1.13021031, 1.13342735, 1.14686068, 1.14502975 + , 1.14474747, 1.14084037, 1.16142926], + [ 1.06336033, 1.07365823, 1.08691496, 1.09764846, 1.11669863, 1.11856702 + , 1.09764283, 1.08815849, 1.08044313, 1.09278827, 1.07003204, 1.08398066 + , 1.09831768, 1.09298232, 1.09176125], + [ 0.79772065, 0.78829196, 0.78581151, 0.77615922, 0.77035744, 0.77751194 + , 0.79902974, 0.81437881, 0.80788828, 0.79603865, 0.78966436, 0.79949807 + , 0.80172182, 0.82168155, 0.85587911], + [ 1.0052447, 1.00007696, 1.00475899, 1.00613942, 1.00639561, 1.00162979 + , 0.99860739, 1.00814981, 1.00574316, 0.99030032, 0.97682565, 0.97292596 + , 0.96519561, 0.96173403, 0.95890284], + [ 0.95808419, 0.9382568, 0.9654441, 0.95561201, 0.96987289, 0.96608031 + , 0.99727185, 1.00781194, 1.03484236, 1.05333619, 1.0983263, 1.1704974 + , 1.17025154, 1.18730553, 1.14242645]]) + + self.assertTrue(np.allclose(result, expected)) + self.assertTrue(type(result) == type(expected)) + self.assertTrue(result.shape == expected.shape) + + def test_rebin_data(self): + """Test rebin_data""" + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## = 0.5, 2.5, 4.5, 6.5, 8.5 + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + for i in range(0, 10, 2)]).T + + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) + + ## sample in triple the time (uneven since 10 % 3 = 1): + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## = 1, 4, 7, 9 + ans_odd = np.array([i * np.ones(10, dtype=float) + for i in (1, 4, 7, 9)]).T + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) + + def test_get_prob_dist(self): + """Test get_prob_dist""" + lag_indices = np.array([1, 2, 3, 4]) + unit_indices = np.array([1, 3, 2, 4]) + answer = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) + + self.assertTrue(np.array_equal(result, answer)) + + def test_get_prob_stats(self): + """Test get_prob_stats""" + + probs = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + unit_indices = np.array([1, 3, 2, 4]) + answer_up = np.array([0.04245283, 0.03529412, 0.12376238, 0.]) + answer_down = np.array([0.0754717, 0.09411765, 0.0990099, 0.02352941]) + answer_trend = np.array([-0.03301887 / 0.88207547, -0.05882353 / 0.87058824, 0.02475248 / 0.77722772, -0.02352941 / 0.97647059]) + answer_volatility = np.array([ 0.34221495, 0.33705421, 0.29226542, 0.38834223]) + + result = std.get_prob_stats(probs, unit_indices) + result_up = result[0] + result_down = result[1] + result_trend = result[2] + result_volatility = result[3] + + self.assertTrue(np.allclose(result_up, answer_up)) + self.assertTrue(np.allclose(result_down, answer_down)) + self.assertTrue(np.allclose(result_trend, answer_trend)) + self.assertTrue(np.allclose(result_volatility, answer_volatility)) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 01088b1..876fadc 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.4' +default_version = '0.1.0' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft From 3d99d1f9bf97136b530306780c838eed2f00a5a5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 17:39:57 +0000 Subject: [PATCH 139/183] Add upgrade and downgrade files for 0.1.0 --- release/crankshaft--0.0.4--0.1.0.sql | 258 +++++++++++++++++++++++++++ release/crankshaft--0.1.0--0.0.4.sql | 81 +++++++++ 2 files changed, 339 insertions(+) create mode 100644 release/crankshaft--0.0.4--0.1.0.sql create mode 100644 release/crankshaft--0.1.0--0.0.4.sql diff --git a/release/crankshaft--0.0.4--0.1.0.sql b/release/crankshaft--0.0.4--0.1.0.sql new file mode 100644 index 0000000..5a67163 --- /dev/null +++ b/release/crankshaft--0.0.4--0.1.0.sql @@ -0,0 +1,258 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED FROM SOURCES + +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-------------------------------------------------------------------------------- + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.1.0'::text; +$$ language 'sql' STABLE STRICT; + +-------------------------------------------------------------------------------- + +-- PyAgg stuff +CREATE OR REPLACE FUNCTION + CDB_PyAggS(current_state Numeric[], current_row Numeric[]) + returns NUMERIC[] as $$ + BEGIN + if array_upper(current_state,1) is null then + RAISE NOTICE 'setting state %',array_upper(current_row,1); + current_state[1] = array_upper(current_row,1); + end if; + return array_cat(current_state,current_row) ; + END + $$ LANGUAGE plpgsql; + + +CREATE AGGREGATE CDB_PyAgg(NUMERIC[])( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" +); + +-------------------------------------------------------------------------------- + +-- Segmentation stuff +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = {'n_estimators': n_estimators, + 'max_depth': max_depth, + 'subsample': subsample, + 'learning_rate': learning_rate, + 'min_samples_leaf': min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg(np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC) +AS $$ + from crankshaft.segmentation import create_and_predict_segment + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + return create_and_predict_segment(query,variable_name,target_table, model_params) +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Spatial interpolation + +-- 0: nearest neighbor +-- 1: barymetric +-- 2: IDW + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + output numeric; +BEGIN + EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs; + SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a; + + RETURN output; +END; +$$ +language plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + gs2 geometry[]; + vs2 numeric[]; + g geometry; + vertex geometry[]; + sg numeric; + sa numeric; + sb numeric; + sc numeric; + va numeric; + vb numeric; + vc numeric; + output numeric; +BEGIN + output := -999.999; + -- nearest + IF method = 0 THEN + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v) + SELECT a.v INTO output FROM a ORDER BY point<->a.g LIMIT 1; + RETURN output; + + -- barymetric + ELSIF method = 1 THEN + WITH a as (SELECT unnest(geomin) AS e), + b as (SELECT ST_DelaunayTriangles(ST_Collect(a.e),0.001, 0) AS t FROM a), + c as (SELECT (ST_Dump(t)).geom as v FROM b), + d as (SELECT v FROM c WHERE ST_Within(point, v)) + SELECT v INTO g FROM d; + IF g is null THEN + -- out of the realm of the input data + RETURN -888.888; + END IF; + -- vertex of the selected cell + WITH a AS (SELECT (ST_DumpPoints(g)).geom AS v) + SELECT array_agg(v) INTO vertex FROM a; + + -- retrieve the value of each vertex + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]); + + SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc; + + output := (coalesce(sa,0) * coalesce(va,0) + coalesce(sb,0) * coalesce(vb,0) + coalesce(sc,0) * coalesce(vc,0)) / coalesce(sg); + RETURN output; + + -- IDW + -- p1: limit the number of neighbors, 0->no limit + -- p2: order of distance decay, 0-> order 1 + ELSIF method = 2 THEN + + IF p2 = 0 THEN + p2 := 1; + END IF; + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v), + b as (SELECT a.g, a.v FROM a ORDER BY point<->a.g) + SELECT array_agg(b.g), array_agg(b.v) INTO gs, vs FROM b; + IF p1::integer>0 THEN + gs2:=gs; + vs2:=vs; + FOR i IN 1..p1 + LOOP + gs2 := gs2 || gs[i]; + vs2 := vs2 || vs[i]; + END LOOP; + ELSE + gs2:=gs; + vs2:=vs; + END IF; + + WITH a as (SELECT unnest(gs2) as g, unnest(vs2) as v), + b as ( + SELECT + (1/ST_distance(point, a.g)^p2::integer) as k, + (a.v/ST_distance(point, a.g)^p2::integer) as f + FROM a + ) + SELECT sum(b.f)/sum(b.k) INTO output FROM b; + RETURN output; + + END IF; + + RETURN -777.777; + +END; +$$ +language plpgsql IMMUTABLE; + + +-------------------------------------------------------------------------------- + +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + +CREATE OR REPLACE FUNCTION + CDB_SpatialMarkovTrend ( + subquery TEXT, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) +AS $$ + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; diff --git a/release/crankshaft--0.1.0--0.0.4.sql b/release/crankshaft--0.1.0--0.0.4.sql new file mode 100644 index 0000000..4e53dee --- /dev/null +++ b/release/crankshaft--0.1.0--0.0.4.sql @@ -0,0 +1,81 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED FROM SOURCES + +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-------------------------------------------------------------------------------- + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.4'::text; +$$ language 'sql' STABLE STRICT; + +-------------------------------------------------------------------------------- + +-- PyAgg stuff +DROP FUNCTION CDB_PyAggS(Numeric[], Numeric[]); +DROP AGGREGATE CDB_PyAgg(NUMERIC[]); + +-------------------------------------------------------------------------------- + +-- Segmentation stuff + +DROP FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1); + +DROP FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1); + +-------------------------------------------------------------------------------- + +-- Spatial interpolation + +DROP FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ); + +DROP FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ); + +-------------------------------------------------------------------------------- + +-- Spatial Markov + +DROP FUNCTION + CDB_SpatialMarkovTrend ( + subquery TEXT, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id'); From c8871a55475c7d0e514a9670967e3ee0dee06e8a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 19:52:01 +0200 Subject: [PATCH 140/183] Remove the DEFAULT values in DROP FUNCTION's --- release/crankshaft--0.1.0--0.0.4.sql | 44 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/release/crankshaft--0.1.0--0.0.4.sql b/release/crankshaft--0.1.0--0.0.4.sql index 4e53dee..01bb059 100644 --- a/release/crankshaft--0.1.0--0.0.4.sql +++ b/release/crankshaft--0.1.0--0.0.4.sql @@ -27,22 +27,22 @@ DROP FUNCTION features NUMERIC[], target_features NUMERIC[], target_ids NUMERIC[], - n_estimators INTEGER DEFAULT 1200, - max_depth INTEGER DEFAULT 3, - subsample DOUBLE PRECISION DEFAULT 0.5, - learning_rate DOUBLE PRECISION DEFAULT 0.01, - min_samples_leaf INTEGER DEFAULT 1); + n_estimators INTEGER, + max_depth INTEGER, + subsample DOUBLE PRECISION, + learning_rate DOUBLE PRECISION, + min_samples_leaf INTEGER); DROP FUNCTION CDB_CreateAndPredictSegment ( query TEXT, variable_name TEXT, target_table TEXT, - n_estimators INTEGER DEFAULT 1200, - max_depth INTEGER DEFAULT 3, - subsample DOUBLE PRECISION DEFAULT 0.5, - learning_rate DOUBLE PRECISION DEFAULT 0.01, - min_samples_leaf INTEGER DEFAULT 1); + n_estimators INTEGER, + max_depth INTEGER, + subsample DOUBLE PRECISION, + learning_rate DOUBLE PRECISION, + min_samples_leaf INTEGER); -------------------------------------------------------------------------------- @@ -51,18 +51,18 @@ DROP FUNCTION DROP FUNCTION CDB_SpatialInterpolation( IN query text, IN point geometry, - IN method integer DEFAULT 1, - IN p1 numeric DEFAULT 0, - IN p2 numeric DEFAULT 0 + IN method integer, + IN p1 numeric, + IN p2 numeric ); DROP FUNCTION CDB_SpatialInterpolation( IN geomin geometry[], IN colin numeric[], IN point geometry, - IN method integer DEFAULT 1, - IN p1 numeric DEFAULT 0, - IN p2 numeric DEFAULT 0 + IN method integer, + IN p1 numeric, + IN p2 numeric ); -------------------------------------------------------------------------------- @@ -73,9 +73,9 @@ DROP FUNCTION CDB_SpatialMarkovTrend ( subquery TEXT, time_cols TEXT[], - num_classes INT DEFAULT 7, - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id'); + num_classes INT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT); From c21fcdf69a29d7363ee8c667d5c3f5503b0600d6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 29 Jun 2016 20:01:40 +0200 Subject: [PATCH 141/183] Drop objects in reverse order in downgrade script Otherwise it fails downgrading, despite it is supposed to run everything in the context of a transaction: ``` tests=# alter extension crankshaft update to '0.0.4'; ERROR: cannot drop function cdb_pyaggs(numeric[],numeric[]) because other objects depend on it DETAIL: extension crankshaft depends on function cdb_pyaggs(numeric[],numeric[]) HINT: Use DROP ... CASCADE to drop the dependent objects too. ``` --- release/crankshaft--0.1.0--0.0.4.sql | 112 +++++++++++++-------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/release/crankshaft--0.1.0--0.0.4.sql b/release/crankshaft--0.1.0--0.0.4.sql index 01bb059..983dbce 100644 --- a/release/crankshaft--0.1.0--0.0.4.sql +++ b/release/crankshaft--0.1.0--0.0.4.sql @@ -3,8 +3,6 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION crankshaft" to load this file. \quit --------------------------------------------------------------------------------- - -- Version number of the extension release CREATE OR REPLACE FUNCTION cdb_crankshaft_version() RETURNS text AS $$ @@ -13,60 +11,6 @@ $$ language 'sql' STABLE STRICT; -------------------------------------------------------------------------------- --- PyAgg stuff -DROP FUNCTION CDB_PyAggS(Numeric[], Numeric[]); -DROP AGGREGATE CDB_PyAgg(NUMERIC[]); - --------------------------------------------------------------------------------- - --- Segmentation stuff - -DROP FUNCTION - CDB_CreateAndPredictSegment( - target NUMERIC[], - features NUMERIC[], - target_features NUMERIC[], - target_ids NUMERIC[], - n_estimators INTEGER, - max_depth INTEGER, - subsample DOUBLE PRECISION, - learning_rate DOUBLE PRECISION, - min_samples_leaf INTEGER); - -DROP FUNCTION - CDB_CreateAndPredictSegment ( - query TEXT, - variable_name TEXT, - target_table TEXT, - n_estimators INTEGER, - max_depth INTEGER, - subsample DOUBLE PRECISION, - learning_rate DOUBLE PRECISION, - min_samples_leaf INTEGER); - --------------------------------------------------------------------------------- - --- Spatial interpolation - -DROP FUNCTION CDB_SpatialInterpolation( - IN query text, - IN point geometry, - IN method integer, - IN p1 numeric, - IN p2 numeric - ); - -DROP FUNCTION CDB_SpatialInterpolation( - IN geomin geometry[], - IN colin numeric[], - IN point geometry, - IN method integer, - IN p1 numeric, - IN p2 numeric - ); - --------------------------------------------------------------------------------- - -- Spatial Markov DROP FUNCTION @@ -79,3 +23,59 @@ DROP FUNCTION permutations INT, geom_col TEXT, id_col TEXT); + + +-------------------------------------------------------------------------------- + +-- Spatial interpolation + +DROP FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer, + IN p1 numeric, + IN p2 numeric + ); + +DROP FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer, + IN p1 numeric, + IN p2 numeric + ); + +-------------------------------------------------------------------------------- + +-- Segmentation stuff + +DROP FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER, + max_depth INTEGER, + subsample DOUBLE PRECISION, + learning_rate DOUBLE PRECISION, + min_samples_leaf INTEGER); + +DROP FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER, + max_depth INTEGER, + subsample DOUBLE PRECISION, + learning_rate DOUBLE PRECISION, + min_samples_leaf INTEGER); + +-------------------------------------------------------------------------------- + +-- PyAgg stuff + +DROP AGGREGATE CDB_PyAgg(NUMERIC[]); +DROP FUNCTION CDB_PyAggS(Numeric[], Numeric[]); \ No newline at end of file From 966dd4268a05b854394d4350310e3e586eb0f59c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 15:24:21 +0200 Subject: [PATCH 142/183] Create role publicuser if it does not exist --- src/pg/test/expected/01_install_test.out | 13 +++++++++++++ src/pg/test/sql/01_install_test.sql | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index e84a48a..79ff047 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,5 +1,18 @@ -- Install dependencies CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; +-- Create role publicuser if it does not exist +DO +$$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'publicuser') THEN + + CREATE ROLE publicuser LOGIN; + END IF; +END +$$ LANGUAGE plpgsql; -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index bbce805..98d7db4 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -2,5 +2,19 @@ CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; +-- Create role publicuser if it does not exist +DO +$$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'publicuser') THEN + + CREATE ROLE publicuser LOGIN; + END IF; +END +$$ LANGUAGE plpgsql; + -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; From c48f9b67b77b4616a4ebb15734ff1ecb0aca66fe Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 16:26:35 +0200 Subject: [PATCH 143/183] First attempt with a .travis.yml file --- .travis.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b2514fc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,43 @@ +language: python + +before_install: + - sudo apt-get -y install make + + - sudo apt-get -y install python-pip + + - sudo apt-get -y install python-software-properties + - sudo add-apt-repository -y ppa:cartodb/sci + - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5 + - sudo add-apt-repository -y ppa:cartodb/gis + - sudo apt-get update + + - sudo apt-get -y install python-joblib=0.8.3-1-cdb1 + - sudo apt-get -y install python-numpy=1:1.6.1-6ubuntu1 + + # Install pysal + - sudo pip install -I pysal==1.11.2 + + - sudo apt-get -y install python-scipy=0.14.0-2-cdb6 + - sudo apt-get -y --no-install-recommends install python-sklearn-lib=0.14.1-3-cdb2 + - sudo apt-get -y --no-install-recommends install python-sklearn=0.14.1-3-cdb2 + - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 + + # Install postgres db and build deps + - sudo apt-get -y install \ + postgresql-9.5 \ + postgresql-server-dev-9.5 \ + postgresql-plpython-9.5 \ + postgresql-9.5-postgis-2.2 \ + postgresql-9.5-postgis-scripts + + # configure it to accept local connections from postgres + - echo -e "# TYPE DATABASE USER ADDRESS METHOD +local all postgres trust +local all all trust +host all all 127.0.0.1/32 trust" \ + | sudo tee /etc/postgresql/9.5/main/pg_hba.conf + - sudo service postgresql reload + +script: + - sudo make install + - make test From 45653a850ede7d997ee423819bfe1bb84172e591 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 16:34:01 +0200 Subject: [PATCH 144/183] Fix travis script by using a single line --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2514fc..4d5da0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,11 +31,8 @@ before_install: postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres - - echo -e "# TYPE DATABASE USER ADDRESS METHOD -local all postgres trust -local all all trust -host all all 127.0.0.1/32 trust" \ - | sudo tee /etc/postgresql/9.5/main/pg_hba.conf + - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ + | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - sudo service postgresql reload script: From f2b5c788c06ab8b2c6dab6d1446b00897011dd26 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 16:48:42 +0200 Subject: [PATCH 145/183] Avoid backslash multiline syntax --- .travis.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d5da0a..0dc8bde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,12 +23,11 @@ before_install: - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 # Install postgres db and build deps - - sudo apt-get -y install \ - postgresql-9.5 \ - postgresql-server-dev-9.5 \ - postgresql-plpython-9.5 \ - postgresql-9.5-postgis-2.2 \ - postgresql-9.5-postgis-scripts + - sudo apt-get -y install postgresql-9.5 + - sudo apt-get -y install postgresql-server-dev-9.5 + - sudo apt-get -y install postgresql-plpython-9.5 + - sudo apt-get -y install postgresql-9.5-postgis-2.2 + - sudo apt-get -y install postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ From 1234bfeb9df357314b5f88a516ed12ab7d1fc10a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 17:11:16 +0200 Subject: [PATCH 146/183] Use c instead of python as language Otherwise it installs a different version of python. We want to test the exact dependencies whenever possible. --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0dc8bde..61a2d51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ -language: python +language: c before_install: - - sudo apt-get -y install make - - sudo apt-get -y install python-pip - sudo apt-get -y install python-software-properties From 3181c51637570f408aa8fcc09d0a8e01dfeeac8f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 17:15:41 +0200 Subject: [PATCH 147/183] Install all postgres packages at once Install all postgres packages at once in order to avoid package configuration issues. --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61a2d51..e03478b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,11 +21,7 @@ before_install: - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 # Install postgres db and build deps - - sudo apt-get -y install postgresql-9.5 - - sudo apt-get -y install postgresql-server-dev-9.5 - - sudo apt-get -y install postgresql-plpython-9.5 - - sudo apt-get -y install postgresql-9.5-postgis-2.2 - - sudo apt-get -y install postgresql-9.5-postgis-scripts + - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ From 57aa28ee5cb1e0a0f250ef2bb0b428e24ed8a659 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 17:53:07 +0200 Subject: [PATCH 148/183] Stop postgresql default instance Stop travis postgres default instance before trying to install postgres 9.5. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e03478b..3ec2232 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ before_install: - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 # Install postgres db and build deps + - sudo /etc/init.d/postgresql stop # stop travis default instance - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres From d078a878906f07a3189e286513ae2673284f5958 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 18:09:51 +0200 Subject: [PATCH 149/183] Check port and restart pg9.5 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3ec2232..35df7ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,8 @@ before_install: # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - - sudo service postgresql reload + - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port + - sudo /etc/init.d/postgresql restart 9.5 script: - sudo make install From 60b9f9bd0e29fa2534a57723e1a4bf1a600a9d30 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 18:23:27 +0200 Subject: [PATCH 150/183] Add a couple of lines to diagnose pg port issue --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 35df7ad..2f21906 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,13 +21,14 @@ before_install: - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 # Install postgres db and build deps + - dpkg -l | grep postgres # TODO: remove this line - sudo /etc/init.d/postgresql stop # stop travis default instance - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port + - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port # TODO: remove this line - sudo /etc/init.d/postgresql restart 9.5 script: From 1abb7d669db8fdc63a42da85de7a0ab3ec9e9ce9 Mon Sep 17 00:00:00 2001 From: abelvm Date: Mon, 8 Aug 2016 12:36:44 -0400 Subject: [PATCH 151/183] recommit --- doc/07_gravity.md | 78 +++++++++++++++ src/pg/sql/07_gravity.sql | 115 +++++++++++++++++++++++ src/pg/test/expected/07_gravity_test.out | 11 +++ src/pg/test/sql/07_gravity_test.sql | 21 +++++ 4 files changed, 225 insertions(+) create mode 100644 doc/07_gravity.md create mode 100644 src/pg/sql/07_gravity.sql create mode 100644 src/pg/test/expected/07_gravity_test.out create mode 100644 src/pg/test/sql/07_gravity_test.sql diff --git a/doc/07_gravity.md b/doc/07_gravity.md new file mode 100644 index 0000000..e4e439e --- /dev/null +++ b/doc/07_gravity.md @@ -0,0 +1,78 @@ +## Gravity Model + +Gravity Models are derived from Newton's Law of Gravity and are used to predict the interaction between a group of populated areas (sources) and a specific target among a group of potential targets, in terms of an attraction factor (weight) + +**CDB_Gravity** is based on the model defined in *Huff's Law of Shopper attraction (1963)* + +### CDB_Gravity(t_id bigint[], t_geom geometry[], t_weight numeric[], s_id bigint[], s_geom geometry[], s_pop numeric[], target bigint, radius integer, minval numeric DEFAULT -10e307) + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| t_id | bigint[] | Array of targets ID | +| t_geom | geometry[] | Array of targets' geometries | +| t_weight | numeric[] | Array of targets's weights | +| s_id | bigint[] | Array of sources ID | +| s_geom | geometry[] | Array of sources' geometries | +| s_pop | numeric[] | Array of sources's population | +| target | bigint | ID of the target under study | +| radius | integer | Radius in meters around the target under study that will be taken into account| +| minval (optional) | numeric | Lowest accepted value of weight, defaults to numeric min_value | + +### CDB_Gravity( target_query text, weight_column text, source_query text, pop_column text, target bigint, radius integer, minval numeric DEFAULT -10e307) + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| target_query | text | Query that defines targets | +| weight_column | text | Column name of weights | +| source_query | text | Query that defines sources | +| pop_column | text | Column name of population | +| target | bigint | cartodb_id of the target under study | +| radius | integer | Radius in meters around the target under study that will be taken into account| +| minval (optional) | numeric | Lowest accepted value of weight, defaults to numeric min_value | + + +### Returns + +| Column Name | Type | Description | +|-------------|------|-------------| +| the_geom | geometry | Geometries of the sources within the radius | +| source_id | bigint | ID of the source | +| target_id | bigint | Target ID from input | +| dist | numeric | Distance in meters source to target (if not points, distance between centroids) | +| h | numeric | Probability of patronage | +| hpop | numeric | Patronaging population | + + +#### Example Usage + +```sql +with t as ( +SELECT + array_agg(cartodb_id::bigint) as id, + array_agg(the_geom) as g, + array_agg(coalesce(gla,0)::numeric) as w +FROM + abel.centros_comerciales_de_madrid +WHERE not no_cc +), +s as ( +SELECT + array_agg(cartodb_id::bigint) as id, + array_agg(center) as g, + array_agg(coalesce(t1_1, 0)::numeric) as p +FROM + sscc_madrid +) +select + g.the_geom, + trunc(g.h,2) as h, + round(g.hpop) as hpop, + trunc(g.dist/1000,2) as dist_km +FROM t, s, CDB_Gravity1(t.id, t.g, t.w, s.id, s.g, s.p, newmall_ID, 100000, 5000) g +``` + + diff --git a/src/pg/sql/07_gravity.sql b/src/pg/sql/07_gravity.sql new file mode 100644 index 0000000..47e5b8e --- /dev/null +++ b/src/pg/sql/07_gravity.sql @@ -0,0 +1,115 @@ +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN target_query text, + IN weight_column text, + IN source_query text, + IN pop_column text, + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_id bigint[]; + t_geom geometry[]; + t_weight numeric[]; + s_id bigint[]; + s_geom geometry[]; + s_pop numeric[]; +BEGIN + EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight; + EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop; + RETURN QUERY + SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN t_id bigint[], + IN t_geom geometry[], + IN t_weight numeric[], + IN s_id bigint[], + IN s_geom geometry[], + IN s_pop numeric[], + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_type text; + s_type text; + t_center geometry[]; + s_center geometry[]; +BEGIN + t_type := GeometryType(t_geom[1]); + s_type := GeometryType(s_geom[1]); + IF t_type = 'POINT' THEN + t_center := t_geom; + ELSE + WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp; + END IF; + IF s_type = 'POINT' THEN + s_center := s_geom; + ELSE + WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp; + END IF; + RETURN QUERY + with target0 as( + SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td + ), + source0 as( + SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp + ), + prev0 as( + SELECT + source0.sg, + source0.sd as sourc_id, + coalesce(source0.sp,0) as sp, + target.td as targ_id, + coalesce(target.tw,0) as tw, + GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance + FROM source0 + CROSS JOIN LATERAL + ( + SELECT + * + FROM target0 + WHERE tw > minval + AND ST_DWithin(geography(source0.sc), geography(tc), radius) + ) AS target + ), + deno as( + SELECT + sourc_id, + sum(tw/distance) as h_deno + FROM + prev0 + GROUP BY sourc_id + ) + SELECT + p.sg as the_geom, + p.sourc_id as source_id, + p.targ_id as target_id, + case when p.distance > 1 then p.distance else 0.0 end as dist, + 100*(p.tw/p.distance)/d.h_deno as h, + p.sp*(p.tw/p.distance)/d.h_deno as hpop + FROM + prev0 p, + deno d + WHERE + p.targ_id = target AND + p.sourc_id = d.sourc_id; +END; +$$ language plpgsql; diff --git a/src/pg/test/expected/07_gravity_test.out b/src/pg/test/expected/07_gravity_test.out new file mode 100644 index 0000000..c101b24 --- /dev/null +++ b/src/pg/test/expected/07_gravity_test.out @@ -0,0 +1,11 @@ + the_geom | h | hpop | dist +--------------------------------------------+-------------------------+--------------------------+---------------- + 01010000001361C3D32B650140DD24068195B34440 | 1.51078258369747945249 | 12.08626066957983561994 | 4964.714459152 + 01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 | 99.955141922 + 0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894711274639196 | 382.23197368267647835174 | 2488.330566505 + 010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870080995954879 | 176.77079350404979774397 | 4359.370460594 + 010100000052B81E85EB510140355EBA490CB24440 | 33.12290506987740864904 | 132.49162027950963459615 | 3703.664449828 + 0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754279248087849 | 196.35755262837744263547 | 2512.092358644 + 01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 | 2926.25725244 + 0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 | 53.54905726651871279586 | 3744.515577777 +(8 rows) diff --git a/src/pg/test/sql/07_gravity_test.sql b/src/pg/test/sql/07_gravity_test.sql new file mode 100644 index 0000000..a86bb23 --- /dev/null +++ b/src/pg/test/sql/07_gravity_test.sql @@ -0,0 +1,21 @@ +WITH t AS ( + SELECT + ARRAY[1,2,3] AS id, + ARRAY[7.0,8.0,3.0] AS w, + ARRAY[ST_GeomFromText('POINT(2.1744 41.4036)'),ST_GeomFromText('POINT(2.1228 41.3809)'),ST_GeomFromText('POINT(2.1511 41.3742)')] AS g +), +s AS ( + SELECT + ARRAY[10,20,30,40,50,60,70,80] AS id, + ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS p, + ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g +) +SELECT + g.the_geom, + g.h, + g.hpop, + g.dist +FROM + t, + s, + CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g; From 3ef7c9f62eac6b1b15ee1a6b16827f21933ba029 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 18:41:19 +0200 Subject: [PATCH 152/183] Run the tests against the right PG9.5 port --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f21906..e54e2ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,16 +21,17 @@ before_install: - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 # Install postgres db and build deps - - dpkg -l | grep postgres # TODO: remove this line - sudo /etc/init.d/postgresql stop # stop travis default instance - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port # TODO: remove this line - sudo /etc/init.d/postgresql restart 9.5 + # save the postgres port for later usage + - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port | grep -oh '[0-9]*' > .pg95.port + script: - sudo make install - - make test + - PGPORT=$(cat .pg95.port) make test From 491eeb34d86a0215fb1da8b78b6f86450a2dc096 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 18:52:48 +0200 Subject: [PATCH 153/183] Show the regression.diffs in case of failure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e54e2ac..d63be49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,4 +34,4 @@ before_install: script: - sudo make install - - PGPORT=$(cat .pg95.port) make test + - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } From 02a20566dca9a624674cff3cdc9f5b7170771f46 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Mon, 8 Aug 2016 19:12:55 +0200 Subject: [PATCH 154/183] Force instalation of libgeos-3.5.0 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d63be49..4ffee4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ before_install: - sudo apt-get -y --no-install-recommends install python-sklearn=0.14.1-3-cdb2 - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 + # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis) + - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2 + # Install postgres db and build deps - sudo /etc/init.d/postgresql stop # stop travis default instance - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts From 14b44c358fc70c525f5614ed77f0fcedc8841f07 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 10:36:03 +0200 Subject: [PATCH 155/183] Add postgis version diagnosing traces --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4ffee4e..8d1056a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,6 @@ before_install: script: - sudo make install + - dpkg -l | grep -i postgis + - PGPORT=$(cat .pg95.port) psql -U postgres -c "SELECT PostGIS_Version();" - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } From 2362e51c10fb8f3b56618d257c3b0f5b4feb839f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 10:46:23 +0200 Subject: [PATCH 156/183] Add postgis version diagnosing traces --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d1056a..2c42bbd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,5 +38,5 @@ before_install: script: - sudo make install - dpkg -l | grep -i postgis - - PGPORT=$(cat .pg95.port) psql -U postgres -c "SELECT PostGIS_Version();" + - PGPORT=$(cat .pg95.port) psql -U postgres -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'postgis';" - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } From 955f25cdae97245c240eb2e0d4646901bc8de054 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 11:04:33 +0200 Subject: [PATCH 157/183] Force installation of postgis specific version --- src/pg/test/expected/01_install_test.out | 2 +- src/pg/test/sql/01_install_test.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index 79ff047..c8a763e 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,6 +1,6 @@ -- Install dependencies CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; +CREATE EXTENSION postgis VERSION '2.2.2'; -- Create role publicuser if it does not exist DO $$ diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index 98d7db4..c90ea59 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,6 +1,6 @@ -- Install dependencies CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; +CREATE EXTENSION postgis VERSION '2.2.2'; -- Create role publicuser if it does not exist DO From 927d66911eaeebca17a6dcc761352392df6f9e9e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 11:04:48 +0200 Subject: [PATCH 158/183] Remove traces from travis --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c42bbd..4ffee4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,4 @@ before_install: script: - sudo make install - - dpkg -l | grep -i postgis - - PGPORT=$(cat .pg95.port) psql -U postgres -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'postgis';" - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } From a636d284573d90a1ba61201932293be3c8c45243 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 11:36:51 +0200 Subject: [PATCH 159/183] Install specific postgres versions This is meant to solve the issue with postgis and the error message "GEOS 3.4 or higher required" as that error is compiled statically into postgis. That basically means the build was using a wrong postgis binary. --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4ffee4e..52f0f52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,11 @@ before_install: # Install postgres db and build deps - sudo /etc/init.d/postgresql stop # stop travis default instance - - sudo apt-get -y install postgresql-9.5 postgresql-server-dev-9.5 postgresql-plpython-9.5 postgresql-9.5-postgis-2.2 postgresql-9.5-postgis-scripts + - sudo apt-get -y install postgresql-9.5=9.5.2-2ubuntu1 + - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-2ubuntu1 + - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-2ubuntu1 + - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2 + - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2 # configure it to accept local connections from postgres - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ From 6de5fb10a0e61176e94973459b8a81490cc99408 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 11:50:35 +0200 Subject: [PATCH 160/183] Add gis-testing ppa to the mix Seems like the production postgis 2.2.2 is only present in gis-testing for the moment. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 52f0f52..831c11b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ before_install: - sudo add-apt-repository -y ppa:cartodb/sci - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5 - sudo add-apt-repository -y ppa:cartodb/gis + - sudo add-apt-repository -y ppa:cartodb/gis-testing - sudo apt-get update - sudo apt-get -y install python-joblib=0.8.3-1-cdb1 From c3afdbff4bdfc8a7a72a4aba2b9f207dfea30cf5 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 12:05:49 +0200 Subject: [PATCH 161/183] Remove all the travis pg/postgis stuff --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 831c11b..b9196cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,13 @@ before_install: # Install postgres db and build deps - sudo /etc/init.d/postgresql stop # stop travis default instance + - sudo apt-get -y remove --purge postgresql-9.1 + - sudo apt-get -y remove --purge postgresql-9.2 + - sudo apt-get -y remove --purge postgresql-9.3 + - sudo apt-get -y remove --purge postgresql-9.4 + - sudo apt-get -y remove --purge postgis + - sudo apt-get -y autoremove + - sudo apt-get -y install postgresql-9.5=9.5.2-2ubuntu1 - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-2ubuntu1 - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-2ubuntu1 From 33aff6a744f848135241f2d05a6d8da623e31fe6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 13:19:41 +0200 Subject: [PATCH 162/183] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ff9090..ee72c18 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# crankshaft +# crankshaft [![Build Status](https://travis-ci.org/CartoDB/crankshaft.svg?branch=develop)](https://travis-ci.org/CartoDB/crankshaft) CartoDB Spatial Analysis extension for PostgreSQL. From 58ef79fb6a9d3fc055cb43d03630e0a3101d9aa6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 15:42:01 +0200 Subject: [PATCH 163/183] Script to check whether a branch is up to date --- .travis.yml | 1 + check-up-to-date-with-master.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100755 check-up-to-date-with-master.sh diff --git a/.travis.yml b/.travis.yml index b9196cb..d3eb863 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: c before_install: + - ./check-up-to-date-with-master.sh - sudo apt-get -y install python-pip - sudo apt-get -y install python-software-properties diff --git a/check-up-to-date-with-master.sh b/check-up-to-date-with-master.sh new file mode 100755 index 0000000..af4b8b4 --- /dev/null +++ b/check-up-to-date-with-master.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Add remote-master +git remote add -t master remote-master https://github.com/CartoDB/crankshaft.git + +# Fetch master reference +git fetch --depth=1 remote-master master + +# Compare HEAD with master +# NOTE: travis by default uses --depth=50 so we are actually checking that the tip +# of the branch is no more than 50 commits away from master as well. +git rev-list HEAD | grep $(git rev-parse remote-master/master) || + { echo "Your branch is not up to date with latest release"; + echo "Please update it by running the following:"; + echo " git fetch && git merge origin/develop"; + false; } From 84d3238d593f85567fa9f11e180fe69243877fa1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 15:44:36 +0200 Subject: [PATCH 164/183] Split installation into its own phase --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d3eb863..9409590 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,8 @@ before_install: # save the postgres port for later usage - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port | grep -oh '[0-9]*' > .pg95.port -script: +install: - sudo make install + +script: - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } From a148fc7d899201a37d67eac31983a65371c3ad7e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 9 Aug 2016 15:45:45 +0200 Subject: [PATCH 165/183] Remove PGPORT hack It is no longer needed as pg9.5 is installed in its standard port. --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9409590..cfd9c52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,11 +45,8 @@ before_install: | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - sudo /etc/init.d/postgresql restart 9.5 - # save the postgres port for later usage - - sudo cat /etc/postgresql/9.5/main/postgresql.conf | grep ^port | grep -oh '[0-9]*' > .pg95.port - install: - sudo make install script: - - PGPORT=$(cat .pg95.port) make test || { cat src/pg/test/regression.diffs; false; } + - make test || { cat src/pg/test/regression.diffs; false; } From 448562f53efb6b1d6a62c4b84b4e79f1dc2da86f Mon Sep 17 00:00:00 2001 From: abelvm Date: Tue, 9 Aug 2016 12:52:37 -0400 Subject: [PATCH 166/183] check tests --- src/pg/test/sql/07_gravity_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/07_gravity_test.sql b/src/pg/test/sql/07_gravity_test.sql index a86bb23..db0cbd6 100644 --- a/src/pg/test/sql/07_gravity_test.sql +++ b/src/pg/test/sql/07_gravity_test.sql @@ -18,4 +18,4 @@ SELECT FROM t, s, - CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g; + crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g; From c4b2ae91bf32ca78c882f255ddd8ccfa7ebfc1c5 Mon Sep 17 00:00:00 2001 From: abelvm Date: Tue, 9 Aug 2016 12:57:50 -0400 Subject: [PATCH 167/183] check tests --- src/pg/test/sql/07_gravity_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/test/sql/07_gravity_test.sql b/src/pg/test/sql/07_gravity_test.sql index db0cbd6..e6a9301 100644 --- a/src/pg/test/sql/07_gravity_test.sql +++ b/src/pg/test/sql/07_gravity_test.sql @@ -18,4 +18,4 @@ SELECT FROM t, s, - crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g; + cdb_crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g; From 244cf65617dfe5612e4efa826889f8b9157c6039 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 10:38:05 +0200 Subject: [PATCH 168/183] Fix gravity tests --- src/pg/test/expected/07_gravity_test.out | 5 ++++- src/pg/test/sql/07_gravity_test.sql | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pg/test/expected/07_gravity_test.out b/src/pg/test/expected/07_gravity_test.out index c101b24..064f3c5 100644 --- a/src/pg/test/expected/07_gravity_test.out +++ b/src/pg/test/expected/07_gravity_test.out @@ -1,4 +1,6 @@ - the_geom | h | hpop | dist +SET client_min_messages TO WARNING; +\set ECHO none + the_geom | h | hpop | dist --------------------------------------------+-------------------------+--------------------------+---------------- 01010000001361C3D32B650140DD24068195B34440 | 1.51078258369747945249 | 12.08626066957983561994 | 4964.714459152 01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 | 99.955141922 @@ -9,3 +11,4 @@ 01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 | 2926.25725244 0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 | 53.54905726651871279586 | 3744.515577777 (8 rows) + diff --git a/src/pg/test/sql/07_gravity_test.sql b/src/pg/test/sql/07_gravity_test.sql index e6a9301..c0db940 100644 --- a/src/pg/test/sql/07_gravity_test.sql +++ b/src/pg/test/sql/07_gravity_test.sql @@ -1,3 +1,6 @@ +SET client_min_messages TO WARNING; +\set ECHO none + WITH t AS ( SELECT ARRAY[1,2,3] AS id, From 34c9a7d4cfa9988d0c6ae68ba3d323124c3de189 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 13:40:30 +0200 Subject: [PATCH 169/183] First version of the compat check script --- check-compatibility.sh | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 check-compatibility.sh diff --git a/check-compatibility.sh b/check-compatibility.sh new file mode 100755 index 0000000..545f99f --- /dev/null +++ b/check-compatibility.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +export PGUSER=postgres + +DBNAME=crankshaft_compatcheck + +function die { + echo $1 + exit -1 +} + +# Create fresh DB +psql -c "CREATE DATABASE $DBNAME;" || die "Could not create DB" + +# Hook for cleanup +function cleanup { + psql -c "DROP DATABASE IF EXISTS crankshaft_compatcheck;" +} +trap cleanup EXIT + +# Deploy previous release +(cd src/py && sudo make deploy RUN_OPTIONS="--no-deps") || die "Could not deploy python extension" +(cd src/pg && sudo make deploy) || die " Could not deploy last release" +psql -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'crankshaft';" + +# Install in the fresh DB +psql $DBNAME <<'EOF' +-- Install dependencies +CREATE EXTENSION plpythonu; +CREATE EXTENSION postgis VERSION '2.2.2'; + +-- Create role publicuser if it does not exist +DO +$$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'publicuser') THEN + + CREATE ROLE publicuser LOGIN; + END IF; +END +$$ LANGUAGE plpgsql; + +-- Install the default version +CREATE EXTENSION crankshaft; +\dx +EOF + +# TODO save public functions and signatures + +# Check it can be upgraded +psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" + + + +# TODO check against saved public functions and signatures + + From cdc072b09847b2b99463daaef93dc0b31c46a242 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:38:28 +0200 Subject: [PATCH 170/183] Add upgrade path from release to dev version --- src/pg/.gitignore | 1 + src/pg/Makefile | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pg/.gitignore b/src/pg/.gitignore index b58a014..56825ae 100644 --- a/src/pg/.gitignore +++ b/src/pg/.gitignore @@ -4,3 +4,4 @@ results/ crankshaft--dev.sql crankshaft--dev--current.sql crankshaft--current--dev.sql +crankshaft--*--dev.sql diff --git a/src/pg/Makefile b/src/pg/Makefile index 178ed08..6775aad 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -10,14 +10,15 @@ include ../../Makefile.global # * test runs the tests for the currently generated Development # extension. -DATA = $(EXTENSION)--dev.sql \ - $(EXTENSION)--current--dev.sql \ - $(EXTENSION)--dev--current.sql +DATA = \ + $(EXTENSION)--dev.sql \ + $(EXTENSION)--current--dev.sql \ + $(EXTENSION)--dev--current.sql \ + $(EXTENSION)--$(RELEASE_VERSION)--dev.sql SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) - REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' $(DATA): $(SOURCES_DATA) From abe07166de544ed1ac1d17ffd0a3267be2f14779 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:39:28 +0200 Subject: [PATCH 171/183] Add installation of current dev version --- check-compatibility.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/check-compatibility.sh b/check-compatibility.sh index 545f99f..f6930e7 100755 --- a/check-compatibility.sh +++ b/check-compatibility.sh @@ -50,11 +50,13 @@ EOF # TODO save public functions and signatures +# Deploy current dev branch +make clean-dev || die "Could not clean dev files" +sudo make install || die "Could not deploy current dev branch" + # Check it can be upgraded psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" # TODO check against saved public functions and signatures - - From e74d80f3eab23a2c6663adda5e2d8229c543207d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:48:45 +0200 Subject: [PATCH 172/183] Add compat checks as a build step --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cfd9c52..c3b166e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,3 +50,4 @@ install: script: - make test || { cat src/pg/test/regression.diffs; false; } + - ./check-compatibility.sh From ac51256463909bdf6b0f75141ad67b93c8e1a4f4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 18:12:43 +0200 Subject: [PATCH 173/183] Fix for stale builds --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c3b166e..a165028 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: c +env: + global: + - PAGER=cat + before_install: - ./check-up-to-date-with-master.sh - sudo apt-get -y install python-pip From 4118b57f1f12cece4899156902625d4159ec36c2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 18:56:11 +0200 Subject: [PATCH 174/183] Create aggregates only if they do not exist --- src/pg/sql/04_py_agg.sql | 26 +++++++++++++++++++------- src/pg/sql/11_kmeans.sql | 26 ++++++++++++++++++++------ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/04_py_agg.sql b/src/pg/sql/04_py_agg.sql index c38e323..a3e881b 100644 --- a/src/pg/sql/04_py_agg.sql +++ b/src/pg/sql/04_py_agg.sql @@ -10,10 +10,22 @@ CREATE OR REPLACE FUNCTION END $$ LANGUAGE plpgsql; - -CREATE AGGREGATE CDB_PyAgg(NUMERIC[])( - SFUNC = CDB_PyAggS, - STYPE = Numeric[], - INITCOND = "{}" -); - +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_pyagg' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" + ); + END IF; +END +$$ LANGUAGE plpgsql; diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 125aac3..f20942f 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -41,9 +41,23 @@ BEGIN END $$ LANGUAGE plpgsql; -CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( - SFUNC = CDB_WeightedMeanS, - FINALFUNC = CDB_WeightedMeanF, - STYPE = Numeric[], - INITCOND = "{0.0,0.0,0.0}" -); +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_weightedmean' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" + ); + END IF; +END +$$ LANGUAGE plpgsql; From 4886c187c4352d2fa95d7882fba991d87441885e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 19:30:51 +0200 Subject: [PATCH 175/183] Check function signatures --- check-compatibility.sh | 50 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/check-compatibility.sh b/check-compatibility.sh index f6930e7..966850c 100755 --- a/check-compatibility.sh +++ b/check-compatibility.sh @@ -48,7 +48,26 @@ CREATE EXTENSION crankshaft; \dx EOF -# TODO save public functions and signatures +# Save public function signatures +psql $DBNAME <<'EOF' +CREATE TABLE release_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE + WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4; +EOF # Deploy current dev branch make clean-dev || die "Could not clean dev files" @@ -57,6 +76,33 @@ sudo make install || die "Could not deploy current dev branch" # Check it can be upgraded psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" +# Check against saved public function signatures +psql $DBNAME <<'EOF' +CREATE TABLE dev_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE + WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4; +EOF +echo "Functions in development not in latest release (ok):" +psql $DBNAME -c "SELECT * FROM dev_function_signatures EXCEPT SELECT * FROM release_function_signatures;" -# TODO check against saved public functions and signatures +echo "Functions in latest release not in development (compat issue):" +psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" + +# Fail if there's a signature mismatch / missing functions +psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" | fgrep '(0 rows)' \ + || die "Function signatures changed" From 75b34d29ce6ed2f814c8d551fb0169c3b4053a2c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 11 Aug 2016 18:47:35 +0200 Subject: [PATCH 176/183] Update default_version and NEWS in prep of v0.2.0 --- NEWS.md | 4 ++++ src/pg/crankshaft.control | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2c19570..e78a7e1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +0.2.0 (2016-08-11) +------------------ +* Adds Gravity Model + 0.1.0 (2016-06-29) ------------------ * Adds Spatial Markov function diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 876fadc..6f48fdd 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.1.0' +default_version = '0.2.0' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft From 1bad568fbf3aa754fb8a7fa4904bd35478602652 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 11 Aug 2016 18:49:55 +0200 Subject: [PATCH 177/183] Add files generated with make release v0.2.0 --- release/crankshaft--0.2.0.sql | 827 ++++++++++++++++++ release/crankshaft.control | 2 +- .../0.2.0/crankshaft/crankshaft/__init__.py | 5 + .../crankshaft/clustering/__init__.py | 3 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 262 ++++++ .../crankshaft/pysal_utils/__init__.py | 2 + .../crankshaft/pysal_utils/pysal_utils.py | 188 ++++ .../crankshaft/crankshaft/random_seeds.py | 11 + .../crankshaft/segmentation/__init__.py | 1 + .../crankshaft/segmentation/segmentation.py | 176 ++++ .../space_time_dynamics/__init__.py | 2 + .../crankshaft/space_time_dynamics/markov.py | 189 ++++ release/python/0.2.0/crankshaft/setup.py | 49 ++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../crankshaft/test/fixtures/markov.json | 1 + .../0.2.0/crankshaft/test/fixtures/moran.json | 52 ++ .../crankshaft/test/fixtures/neighbors.json | 54 ++ .../test/fixtures/neighbors_markov.json | 1 + .../python/0.2.0/crankshaft/test/helper.py | 13 + .../python/0.2.0/crankshaft/test/mock_plpy.py | 52 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 + .../crankshaft/test/test_clustering_moran.py | 88 ++ .../0.2.0/crankshaft/test/test_pysal_utils.py | 142 +++ .../crankshaft/test/test_segmentation.py | 64 ++ .../test/test_space_time_dynamics.py | 324 +++++++ 26 files changed, 2564 insertions(+), 1 deletion(-) create mode 100644 release/crankshaft--0.2.0.sql create mode 100644 release/python/0.2.0/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/segmentation/__init__.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/segmentation/segmentation.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/__init__.py create mode 100644 release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/markov.py create mode 100644 release/python/0.2.0/crankshaft/setup.py create mode 100644 release/python/0.2.0/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.2.0/crankshaft/test/fixtures/markov.json create mode 100644 release/python/0.2.0/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.2.0/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.2.0/crankshaft/test/fixtures/neighbors_markov.json create mode 100644 release/python/0.2.0/crankshaft/test/helper.py create mode 100644 release/python/0.2.0/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.2.0/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.2.0/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.2.0/crankshaft/test/test_pysal_utils.py create mode 100644 release/python/0.2.0/crankshaft/test/test_segmentation.py create mode 100644 release/python/0.2.0/crankshaft/test/test_space_time_dynamics.py diff --git a/release/crankshaft--0.2.0.sql b/release/crankshaft--0.2.0.sql new file mode 100644 index 0000000..1cb3087 --- /dev/null +++ b/release/crankshaft--0.2.0.sql @@ -0,0 +1,827 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.2.0'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION + CDB_PyAggS(current_state Numeric[], current_row Numeric[]) + returns NUMERIC[] as $$ + BEGIN + if array_upper(current_state,1) is null then + RAISE NOTICE 'setting state %',array_upper(current_row,1); + current_state[1] = array_upper(current_row,1); + end if; + return array_cat(current_state,current_row) ; + END + $$ LANGUAGE plpgsql; + +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_pyagg' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" + ); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = {'n_estimators': n_estimators, + 'max_depth': max_depth, + 'subsample': subsample, + 'learning_rate': learning_rate, + 'min_samples_leaf': min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg(np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC) +AS $$ + from crankshaft.segmentation import create_and_predict_segment + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + return create_and_predict_segment(query,variable_name,target_table, model_params) +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN target_query text, + IN weight_column text, + IN source_query text, + IN pop_column text, + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_id bigint[]; + t_geom geometry[]; + t_weight numeric[]; + s_id bigint[]; + s_geom geometry[]; + s_pop numeric[]; +BEGIN + EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight; + EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop; + RETURN QUERY + SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN t_id bigint[], + IN t_geom geometry[], + IN t_weight numeric[], + IN s_id bigint[], + IN s_geom geometry[], + IN s_pop numeric[], + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_type text; + s_type text; + t_center geometry[]; + s_center geometry[]; +BEGIN + t_type := GeometryType(t_geom[1]); + s_type := GeometryType(s_geom[1]); + IF t_type = 'POINT' THEN + t_center := t_geom; + ELSE + WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp; + END IF; + IF s_type = 'POINT' THEN + s_center := s_geom; + ELSE + WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp; + END IF; + RETURN QUERY + with target0 as( + SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td + ), + source0 as( + SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp + ), + prev0 as( + SELECT + source0.sg, + source0.sd as sourc_id, + coalesce(source0.sp,0) as sp, + target.td as targ_id, + coalesce(target.tw,0) as tw, + GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance + FROM source0 + CROSS JOIN LATERAL + ( + SELECT + * + FROM target0 + WHERE tw > minval + AND ST_DWithin(geography(source0.sc), geography(tc), radius) + ) AS target + ), + deno as( + SELECT + sourc_id, + sum(tw/distance) as h_deno + FROM + prev0 + GROUP BY sourc_id + ) + SELECT + p.sg as the_geom, + p.sourc_id as source_id, + p.targ_id as target_id, + case when p.distance > 1 then p.distance else 0.0 end as dist, + 100*(p.tw/p.distance)/d.h_deno as h, + p.sp*(p.tw/p.distance)/d.h_deno as hpop + FROM + prev0 p, + deno d + WHERE + p.targ_id = target AND + p.sourc_id = d.sourc_id; +END; +$$ language plpgsql; +-- 0: nearest neighbor +-- 1: barymetric +-- 2: IDW + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + output numeric; +BEGIN + EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs; + SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a; + + RETURN output; +END; +$$ +language plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + gs2 geometry[]; + vs2 numeric[]; + g geometry; + vertex geometry[]; + sg numeric; + sa numeric; + sb numeric; + sc numeric; + va numeric; + vb numeric; + vc numeric; + output numeric; +BEGIN + output := -999.999; + -- nearest + IF method = 0 THEN + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v) + SELECT a.v INTO output FROM a ORDER BY point<->a.g LIMIT 1; + RETURN output; + + -- barymetric + ELSIF method = 1 THEN + WITH a as (SELECT unnest(geomin) AS e), + b as (SELECT ST_DelaunayTriangles(ST_Collect(a.e),0.001, 0) AS t FROM a), + c as (SELECT (ST_Dump(t)).geom as v FROM b), + d as (SELECT v FROM c WHERE ST_Within(point, v)) + SELECT v INTO g FROM d; + IF g is null THEN + -- out of the realm of the input data + RETURN -888.888; + END IF; + -- vertex of the selected cell + WITH a AS (SELECT (ST_DumpPoints(g)).geom AS v) + SELECT array_agg(v) INTO vertex FROM a; + + -- retrieve the value of each vertex + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]); + + SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc; + + output := (coalesce(sa,0) * coalesce(va,0) + coalesce(sb,0) * coalesce(vb,0) + coalesce(sc,0) * coalesce(vc,0)) / coalesce(sg); + RETURN output; + + -- IDW + -- p1: limit the number of neighbors, 0->no limit + -- p2: order of distance decay, 0-> order 1 + ELSIF method = 2 THEN + + IF p2 = 0 THEN + p2 := 1; + END IF; + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v), + b as (SELECT a.g, a.v FROM a ORDER BY point<->a.g) + SELECT array_agg(b.g), array_agg(b.v) INTO gs, vs FROM b; + IF p1::integer>0 THEN + gs2:=gs; + vs2:=vs; + FOR i IN 1..p1 + LOOP + gs2 := gs2 || gs[i]; + vs2 := vs2 || vs[i]; + END LOOP; + ELSE + gs2:=gs; + vs2:=vs; + END IF; + + WITH a as (SELECT unnest(gs2) as g, unnest(vs2) as v), + b as ( + SELECT + (1/ST_distance(point, a.g)^p2::integer) as k, + (a.v/ST_distance(point, a.g)^p2::integer) as f + FROM a + ) + SELECT sum(b.f)/sum(b.k) INTO output FROM b; + RETURN output; + + END IF; + + RETURN -777.777; + +END; +$$ +language plpgsql IMMUTABLE; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_weightedmean' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" + ); + END IF; +END +$$ LANGUAGE plpgsql; +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + +CREATE OR REPLACE FUNCTION + CDB_SpatialMarkovTrend ( + subquery TEXT, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) +AS $$ + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- input table format: identical to above but in a predictable format +-- Sample function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- 'date_1') + + +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col_min text, +-- time_col_max text, +-- date_format text, -- '_YYYY_MM_DD' +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- +-- -- input table format: +-- -- id | geom | date | measurement +-- -- 1 | Pt1 | 12/3 | 13.2 +-- -- 2 | Pt2 | 11/5 | 11.3 +-- -- 3 | Pt1 | 11/13 | 12.9 +-- -- 4 | Pt3 | 12/19 | 10.1 +-- -- ... +-- +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col text, +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 876fadc..6f48fdd 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.1.0' +default_version = '0.2.0' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.2.0/crankshaft/crankshaft/__init__.py b/release/python/0.2.0/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..4e06bc5 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/__init__.py @@ -0,0 +1,5 @@ +"""Import all modules""" +import crankshaft.random_seeds +import crankshaft.clustering +import crankshaft.space_time_dynamics +import crankshaft.segmentation diff --git a/release/python/0.2.0/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.2.0/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..ed34fe0 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,3 @@ +"""Import all functions from for clustering""" +from moran import * +from kmeans import * diff --git a/release/python/0.2.0/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.2.0/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.2.0/crankshaft/crankshaft/clustering/moran.py b/release/python/0.2.0/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..3282f5f --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,262 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy +from collections import OrderedDict + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr_name), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = OrderedDict([("id_col", id_col), + ("attr1", numerator), + ("attr2", denominator) + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = OrderedDict([("id_col", id_col), + ("numerator", numerator), + ("denominator", denominator), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = OrderedDict([("id_col", id_col), + ("attr1", attr1), + ("attr2", attr2), + ("geom_col", geom_col), + ("subquery", subquery), + ("num_ngbrs", num_ngbrs)]) + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..fdf073b --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1,2 @@ +"""Import all functions for pysal_utils""" +from crankshaft.pysal_utils.pysal_utils import * diff --git a/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..4622925 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,188 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on + translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res dict-like: query results with attributes and neighbors + """ + # if w_type.lower() == 'knn': + # row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + # weights = {x['id']: row_normed_weights for x in query_res} + # else: + # weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + # if len(x['neighbors']) > 0 + # else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + print 'len of neighbors: %d' % len(neighbors) + + built_weight = ps.W(neighbors) + built_weight.transform = 'r' + + return built_weight + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attr_string = "" + template = "i.\"%(col)s\"::numeric As attr%(alias_num)s, " + + if 'time_cols' in params: + ## if markov analysis + attrs = params['time_cols'] + + for idx, val in enumerate(attrs): + attr_string += template % {"col": val, "alias_num": idx + 1} + else: + ## if moran's analysis + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')] + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": params[val], "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Construct where conditions when building neighbors query + Create portion of WHERE clauses for weeding out NULL-valued geometries + Input: dict of params: + {'subquery': ..., + 'numerator': 'data1', + 'denominator': 'data2', + '': ...} + Output: 'idx_replace."data1" IS NOT NULL AND idx_replace."data2" IS NOT NULL' + Input: + {'subquery': ..., + 'time_cols': ['time1', 'time2', 'time3'], + 'etc': ...} + Output: 'idx_replace."time1" IS NOT NULL AND idx_replace."time2" IS NOT + NULL AND idx_replace."time3" IS NOT NULL' + """ + attr_string = [] + template = "idx_replace.\"%s\" IS NOT NULL" + + if 'time_cols' in params: + ## markov where clauses + attrs = params['time_cols'] + # add values to template + for attr in attrs: + attr_string.append(template % attr) + else: + ## moran where clauses + + # get keys + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs', 'subquery')]) + # add values to template + for attr in attrs: + attr_string.append(template % params[attr]) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"%s\" <> 0" % params[attrs[1]]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.2.0/crankshaft/crankshaft/random_seeds.py b/release/python/0.2.0/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..31958cb --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,11 @@ +"""Random seed generator used for non-deterministic functions in crankshaft""" +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.2.0/crankshaft/crankshaft/segmentation/__init__.py b/release/python/0.2.0/crankshaft/crankshaft/segmentation/__init__.py new file mode 100644 index 0000000..b825e85 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/segmentation/__init__.py @@ -0,0 +1 @@ +from segmentation import * diff --git a/release/python/0.2.0/crankshaft/crankshaft/segmentation/segmentation.py b/release/python/0.2.0/crankshaft/crankshaft/segmentation/segmentation.py new file mode 100644 index 0000000..ed61139 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/segmentation/segmentation.py @@ -0,0 +1,176 @@ +""" +Segmentation creation and prediction +""" + +import sklearn +import numpy as np +import plpy +from sklearn.ensemble import GradientBoostingRegressor +from sklearn import metrics +from sklearn.cross_validation import train_test_split + +# Lower level functions +#---------------------- + +def replace_nan_with_mean(array): + """ + Input: + @param array: an array of floats which may have null-valued entries + Output: + array with nans filled in with the mean of the dataset + """ + # returns an array of rows and column indices + indices = np.where(np.isnan(array)) + + # iterate through entries which have nan values + for row, col in zip(*indices): + array[row, col] = np.mean(array[~np.isnan(array[:, col]), col]) + + return array + +def get_data(variable, feature_columns, query): + """ + Fetch data from the database, clean, and package into + numpy arrays + Input: + @param variable: name of the target variable + @param feature_columns: list of column names + @param query: subquery that data is pulled from for the packaging + Output: + prepared data, packaged into NumPy arrays + """ + + columns = ','.join(['array_agg("{col}") As "{col}"'.format(col=col) for col in feature_columns]) + + try: + data = plpy.execute('''SELECT array_agg("{variable}") As target, {columns} FROM ({query}) As a'''.format( + variable=variable, + columns=columns, + query=query)) + except Exception, e: + plpy.error('Failed to access data to build segmentation model: %s' % e) + + # extract target data from plpy object + target = np.array(data[0]['target']) + + # put n feature data arrays into an n x m array of arrays + features = np.column_stack([np.array(data[0][col], dtype=float) for col in feature_columns]) + + return replace_nan_with_mean(target), replace_nan_with_mean(features) + +# High level interface +# -------------------- + +def create_and_predict_segment_agg(target, features, target_features, target_ids, model_parameters): + """ + Version of create_and_predict_segment that works on arrays that come stright form the SQL calling + the function. + + Input: + @param target: The 1D array of lenth NSamples containing the target variable we want the model to predict + @param features: Thw 2D array of size NSamples * NFeatures that form the imput to the model + @param target_ids: A 1D array of target_ids that will be used to associate the results of the prediction with the rows which they come from + @param model_parameters: A dictionary containing parameters for the model. + """ + + clean_target = replace_nan_with_mean(target) + clean_features = replace_nan_with_mean(features) + target_features = replace_nan_with_mean(target_features) + + model, accuracy = train_model(clean_target, clean_features, model_parameters, 0.2) + prediction = model.predict(target_features) + accuracy_array = [accuracy]*prediction.shape[0] + return zip(target_ids, prediction, np.full(prediction.shape, accuracy_array)) + + + +def create_and_predict_segment(query, variable, target_query, model_params): + """ + generate a segment with machine learning + Stuart Lynn + """ + + ## fetch column names + try: + columns = plpy.execute('SELECT * FROM ({query}) As a LIMIT 1 '.format(query=query))[0].keys() + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + ## extract column names to be used in building the segmentation model + feature_columns = set(columns) - set([variable, 'cartodb_id', 'the_geom', 'the_geom_webmercator']) + ## get data from database + target, features = get_data(variable, feature_columns, query) + + model, accuracy = train_model(target, features, model_params, 0.2) + cartodb_ids, result = predict_segment(model, feature_columns, target_query) + accuracy_array = [accuracy]*result.shape[0] + return zip(cartodb_ids, result, accuracy_array) + + +def train_model(target, features, model_params, test_split): + """ + Train the Gradient Boosting model on the provided data and calculate the accuracy of the model + Input: + @param target: 1D Array of the variable that the model is to be trianed to predict + @param features: 2D Array NSamples * NFeatures to use in trining the model + @param model_params: A dictionary of model parameters, the full specification can be found on the + scikit learn page for [GradientBoostingRegressor](http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingRegressor.html) + @parma test_split: The fraction of the data to be withheld for testing the model / calculating the accuray + """ + features_train, features_test, target_train, target_test = train_test_split(features, target, test_size=test_split) + model = GradientBoostingRegressor(**model_params) + model.fit(features_train, target_train) + accuracy = calculate_model_accuracy(model, features, target) + return model, accuracy + +def calculate_model_accuracy(model, features, target): + """ + Calculate the mean squared error of the model prediction + Input: + @param model: model trained from input features + @param features: features to make a prediction from + @param target: target to compare prediction to + Output: + mean squared error of the model prection compared to the target + """ + prediction = model.predict(features) + return metrics.mean_squared_error(prediction, target) + +def predict_segment(model, features, target_query): + """ + Use the provided model to predict the values for the new feature set + Input: + @param model: The pretrained model + @features: A list of features to use in the model prediction (list of column names) + @target_query: The query to run to obtain the data to predict on and the cartdb_ids associated with it. + """ + + batch_size = 1000 + joined_features = ','.join(['"{0}"::numeric'.format(a) for a in features]) + + try: + cursor = plpy.cursor('SELECT Array[{joined_features}] As features FROM ({target_query}) As a'.format( + joined_features=joined_features, + target_query=target_query)) + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + results = [] + + while True: + rows = cursor.fetch(batch_size) + if not rows: + break + batch = np.row_stack([np.array(row['features'], dtype=float) for row in rows]) + + #Need to fix this. Should be global mean. This will cause weird effects + batch = replace_nan_with_mean(batch) + prediction = model.predict(batch) + results.append(prediction) + + try: + cartodb_ids = plpy.execute('''SELECT array_agg(cartodb_id ORDER BY cartodb_id) As cartodb_ids FROM ({0}) As a'''.format(target_query))[0]['cartodb_ids'] + except Exception, e: + plpy.error('Failed to build segmentation model: %s' % e) + + return cartodb_ids, np.concatenate(results) diff --git a/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/__init__.py b/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/__init__.py new file mode 100644 index 0000000..a439286 --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/__init__.py @@ -0,0 +1,2 @@ +"""Import all functions from clustering libraries.""" +from markov import * diff --git a/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/markov.py b/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/markov.py new file mode 100644 index 0000000..bbf524d --- /dev/null +++ b/release/python/0.2.0/crankshaft/crankshaft/space_time_dynamics/markov.py @@ -0,0 +1,189 @@ +""" +Spatial dynamics measurements using Spatial Markov +""" + + +import numpy as np +import pysal as ps +import plpy +import crankshaft.pysal_utils as pu + +def spatial_markov_trend(subquery, time_cols, num_classes=7, + w_type='knn', num_ngbrs=5, permutations=0, + geom_col='the_geom', id_col='cartodb_id'): + """ + Predict the trends of a unit based on: + 1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile) + 2. average class of its neighbors + + Inputs: + @param subquery string: e.g., SELECT the_geom, cartodb_id, + interesting_time_column FROM table_name + @param time_cols list of strings: list of strings of column names + @param num_classes (optional): number of classes to break distribution + of values into. Currently uses quantile bins. + @param w_type string (optional): weight type ('knn' or 'queen') + @param num_ngbrs int (optional): number of neighbors (if knn type) + @param permutations int (optional): number of permutations for test + stats + @param geom_col string (optional): name of column which contains the + geometries + @param id_col string (optional): name of column which has the ids of + the table + + Outputs: + @param trend_up float: probablity that a geom will move to a higher + class + @param trend_down float: probablity that a geom will move to a lower + class + @param trend float: (trend_up - trend_down) / trend_static + @param volatility float: a measure of the volatility based on + probability stddev(prob array) + """ + + if len(time_cols) < 2: + plpy.error('More than one time column needs to be passed') + + qvals = {"id_col": id_col, + "time_cols": time_cols, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + try: + query_result = plpy.execute( + pu.construct_neighbor_query(w_type, qvals) + ) + if len(query_result) == 0: + return zip([None], [None], [None], [None], [None]) + except plpy.SPIError, err: + plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals))) + plpy.error('Query failed, check the input parameters') + return zip([None], [None], [None], [None], [None]) + + ## build weight + weights = pu.get_weight(query_result, w_type) + weights.transform = 'r' + + ## prep time data + t_data = get_time_data(query_result, time_cols) + + plpy.debug('shape of t_data %d, %d' % t_data.shape) + plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape) + plpy.debug('first num elements: %f' % t_data[0, 0]) + + sp_markov_result = ps.Spatial_Markov(t_data, + weights, + k=num_classes, + fixed=False, + permutations=permutations) + + ## get lag classes + lag_classes = ps.Quantiles( + ps.lag_spatial(weights, t_data[:, -1]), + k=num_classes).yb + + ## look up probablity distribution for each unit according to class and lag class + prob_dist = get_prob_dist(sp_markov_result.P, + lag_classes, + sp_markov_result.classes[:, -1]) + + ## find the ups and down and overall distribution of each cell + trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, + sp_markov_result.classes[:, -1]) + + ## output the results + return zip(trend, trend_up, trend_down, volatility, weights.id_order) + +def get_time_data(markov_data, time_cols): + """ + Extract the time columns and bin appropriately + """ + num_attrs = len(time_cols) + return np.array([[x['attr' + str(i)] for x in markov_data] + for i in range(1, num_attrs+1)], dtype=float).transpose() + +## not currently used +def rebin_data(time_data, num_time_per_bin): + """ + Convert an n x l matrix into an (n/m) x l matrix where the values are + reduced (averaged) for the intervening states: + 1 2 3 4 1.5 3.5 + 5 6 7 8 -> 5.5 7.5 + 9 8 7 6 8.5 6.5 + 5 4 3 2 4.5 2.5 + + if m = 2, the 4 x 4 matrix is transformed to a 2 x 4 matrix. + + This process effectively resamples the data at a longer time span n + units longer than the input data. + For cases when there is a remainder (remainder(5/3) = 2), the remaining + two columns are binned together as the last time period, while the + first three are binned together for the first period. + + Input: + @param time_data n x l ndarray: measurements of an attribute at + different time intervals + @param num_time_per_bin int: number of columns to average into a new + column + Output: + ceil(n / m) x l ndarray of resampled time series + """ + + if time_data.shape[1] % num_time_per_bin == 0: + ## if fit is perfect, then use it + n_max = time_data.shape[1] / num_time_per_bin + else: + ## fit remainders into an additional column + n_max = time_data.shape[1] / num_time_per_bin + 1 + + return np.array([time_data[:, num_time_per_bin * i:num_time_per_bin * (i+1)].mean(axis=1) + for i in range(n_max)]).T + +def get_prob_dist(transition_matrix, lag_indices, unit_indices): + """ + Given an array of transition matrices, look up the probability + associated with the arrangements passed + + Input: + @param transition_matrix ndarray[k,k,k]: + @param lag_indices ndarray: + @param unit_indices ndarray: + + Output: + Array of probability distributions + """ + + return np.array([transition_matrix[(lag_indices[i], unit_indices[i])] + for i in range(len(lag_indices))]) + +def get_prob_stats(prob_dist, unit_indices): + """ + get the statistics of the probability distributions + + Outputs: + @param trend_up ndarray(float): sum of probabilities for upward + movement (relative to the unit index of that prob) + @param trend_down ndarray(float): sum of probabilities for downward + movement (relative to the unit index of that prob) + @param trend ndarray(float): difference of upward and downward + movements + """ + + num_elements = len(unit_indices) + trend_up = np.empty(num_elements, dtype=float) + trend_down = np.empty(num_elements, dtype=float) + trend = np.empty(num_elements, dtype=float) + + for i in range(num_elements): + trend_up[i] = prob_dist[i, (unit_indices[i]+1):].sum() + trend_down[i] = prob_dist[i, :unit_indices[i]].sum() + if prob_dist[i, unit_indices[i]] > 0.0: + trend[i] = (trend_up[i] - trend_down[i]) / prob_dist[i, unit_indices[i]] + else: + trend[i] = None + + ## calculate volatility of distribution + volatility = prob_dist.std(axis=1) + + return trend_up, trend_down, trend, volatility diff --git a/release/python/0.2.0/crankshaft/setup.py b/release/python/0.2.0/crankshaft/setup.py new file mode 100644 index 0000000..0b1f2d6 --- /dev/null +++ b/release/python/0.2.0/crankshaft/setup.py @@ -0,0 +1,49 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.2.0', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + # IMPORTANT NOTE: please don't change this line. Instead issue a ticket to systems for evaluation. + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.2.0/crankshaft/test/fixtures/kmeans.json b/release/python/0.2.0/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.2.0/crankshaft/test/fixtures/markov.json b/release/python/0.2.0/crankshaft/test/fixtures/markov.json new file mode 100644 index 0000000..d60e4e0 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/fixtures/markov.json @@ -0,0 +1 @@ +[[0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 0], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 1], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 2], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 3], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 4], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 5], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 6], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 7], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 8], [0.19047619047619049, 0.16, 0.0, 0.32594478059941379, 9], [-0.23529411764705882, 0.0, 0.19047619047619047, 0.31356338348865387, 10], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 11], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 12], [0.027777777777777783, 0.11111111111111112, 0.088888888888888892, 0.30339641183779581, 13], [0.03125, 0.030303030303030304, 0.0, 0.3850273981640871, 14], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 15], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 16], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 17], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 18], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 19], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 20], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 21], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 22], [-0.16666666666666663, 0.18181818181818182, 0.27272727272727271, 0.20246415864836445, 23], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 24], [0.1875, 0.23999999999999999, 0.12, 0.23731835158706122, 25], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 26], [-0.043478260869565216, 0.0, 0.041666666666666664, 0.37950991789118999, 27], [0.22222222222222221, 0.18181818181818182, 0.0, 0.31701083225750354, 28], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 29], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 30], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 31], [0.030303030303030304, 0.078947368421052627, 0.052631578947368418, 0.33560628561957595, 32], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 33], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 34], [0.0, 0.10000000000000001, 0.10000000000000001, 0.30331501776206204, 35], [-0.054054054054054057, 0.0, 0.05128205128205128, 0.37488547451276033, 36], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 37], [-0.22222222222222224, 0.13333333333333333, 0.26666666666666666, 0.22310934040908681, 38], [-0.0625, 0.095238095238095233, 0.14285714285714285, 0.28634850244519822, 39], [0.034482758620689655, 0.0625, 0.03125, 0.35388469167230169, 40], [0.11111111111111112, 0.10000000000000001, 0.0, 0.35213633723318016, 41], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 42], [0.0, 0.0, 0.0, 0.40000000000000002, 43], [0.0, 0.065217391304347824, 0.065217391304347824, 0.33605067580764519, 44], [0.078947368421052641, 0.073170731707317083, 0.0, 0.36451788667842738, 45], [0.052631578947368425, 0.090909090909090912, 0.045454545454545456, 0.33352611505171165, 46], [-0.20512820512820512, 0.0, 0.1702127659574468, 0.32172013908826891, 47]] diff --git a/release/python/0.2.0/crankshaft/test/fixtures/moran.json b/release/python/0.2.0/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.2.0/crankshaft/test/fixtures/neighbors.json b/release/python/0.2.0/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.2.0/crankshaft/test/fixtures/neighbors_markov.json b/release/python/0.2.0/crankshaft/test/fixtures/neighbors_markov.json new file mode 100644 index 0000000..45a20e7 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/fixtures/neighbors_markov.json @@ -0,0 +1 @@ +[{"neighbors": [10, 7, 21, 23, 1], "y1995": 0.87654416055651474, "y1997": 0.85637566664752718, "y1996": 0.8631470006766887, "y1999": 0.84461540228037335, "y1998": 0.84811668329242784, "y2006": 0.86302631339545688, "y2007": 0.86148266513456728, "y2004": 0.86416611731111015, "y2005": 0.87119374831581786, "y2002": 0.85012592862683589, "y2003": 0.8550965633336135, "y2000": 0.83271652434603094, "y2001": 0.83786313566577242, "id": 0, "y2008": 0.86252252380501315, "y2009": 0.86746356478544273}, {"neighbors": [5, 7, 22, 29, 3], "y1995": 0.91889509774542122, "y1997": 0.92333257900976462, "y1996": 0.91757931190043385, "y1999": 0.92552387732371888, "y1998": 0.92517289327379471, "y2006": 0.91706053906277052, "y2007": 0.90139504820726424, "y2004": 0.89815175749309051, "y2005": 0.91832090781161113, "y2002": 0.89431990798552208, "y2003": 0.88924793576523797, "y2000": 0.90746978227271013, "y2001": 0.89830489127332913, "id": 1, "y2008": 0.87897455159080617, "y2009": 0.86216858051752643}, {"neighbors": [11, 8, 13, 18, 17], "y1995": 0.82591007476914713, "y1997": 0.81989792988843901, "y1996": 0.82548595539161707, "y1999": 0.81731522200916285, "y1998": 0.81503235035017918, "y2006": 0.81814804358939286, "y2007": 0.83675961003285626, "y2004": 0.82668195534569056, "y2005": 0.82373723764184559, "y2002": 0.80849979516360859, "y2003": 0.82258550658074148, "y2000": 0.78964559168205917, "y2001": 0.8058444152731008, "id": 2, "y2008": 0.8357419865626442, "y2009": 0.84647177436289112}, {"neighbors": [4, 14, 9, 5, 12], "y1995": 1.0908817638059434, "y1997": 1.0845641754849344, "y1996": 1.0853768890893893, "y1999": 1.098988414417104, "y1998": 1.0841540389418189, "y2006": 1.1316479722785828, "y2007": 1.1295850763954971, "y2004": 1.1139980568106316, "y2005": 1.1216802898290368, "y2002": 1.1116069731657288, "y2003": 1.1088862051501811, "y2000": 1.1450694824791507, "y2001": 1.1215113292620285, "id": 3, "y2008": 1.1137181812756343, "y2009": 1.0993677488645406}, {"neighbors": [14, 3, 9, 31, 12], "y1995": 1.1073144618319228, "y1997": 1.1328363804627946, "y1996": 1.1137394350312471, "y1999": 1.1591002514611153, "y1998": 1.144725587086376, "y2006": 1.1173646811350333, "y2007": 1.1086324218539598, "y2004": 1.1102496406140896, "y2005": 1.11943471361418, "y2002": 1.1475230282561595, "y2003": 1.1184328424005199, "y2000": 1.1689820101690329, "y2001": 1.1721248787169682, "id": 4, "y2008": 1.0964251552643696, "y2009": 1.0776233718455337}, {"neighbors": [29, 1, 22, 7, 4], "y1995": 1.422697571371182, "y1997": 1.4427350196405593, "y1996": 1.4211843379728528, "y1999": 1.4440068434166562, "y1998": 1.4357757095632602, "y2006": 1.4405276647793266, "y2007": 1.4524121586440921, "y2004": 1.4059372049179741, "y2005": 1.4078864636665769, "y2002": 1.4197822680667809, "y2003": 1.3909220829548647, "y2000": 1.4418473669388905, "y2001": 1.4478283203013527, "id": 5, "y2008": 1.4330609762040207, "y2009": 1.4174430982377491}, {"neighbors": [12, 47, 9, 25, 20], "y1995": 1.1307388498039153, "y1997": 1.1107470843142355, "y1996": 1.1311051255854685, "y1999": 1.130881491772973, "y1998": 1.1336463608751246, "y2006": 1.1088003408832796, "y2007": 1.0840170924825394, "y2004": 1.1244623853593112, "y2005": 1.1167100811401538, "y2002": 1.1306293052597198, "y2003": 1.1194498381213465, "y2000": 1.1088813841947593, "y2001": 1.1185662918783175, "id": 6, "y2008": 1.0695920556329086, "y2009": 1.0787522517402164}, {"neighbors": [21, 1, 22, 10, 0], "y1995": 1.0470612357366649, "y1997": 1.0425337165747406, "y1996": 1.0451683097376836, "y1999": 1.0207254480945218, "y1998": 1.0323998680588111, "y2006": 1.0405109962442973, "y2007": 1.0174964540280445, "y2004": 1.0140090547678748, "y2005": 1.0317674181861733, "y2002": 0.99669586934394627, "y2003": 0.99327675611171373, "y2000": 0.99854316295509526, "y2001": 0.98802579761429143, "id": 7, "y2008": 0.9936394033949828, "y2009": 0.98279746069218921}, {"neighbors": [11, 13, 17, 18, 15], "y1995": 0.98996985668705595, "y1997": 0.99491000469481983, "y1996": 1.0014356415938011, "y1999": 1.0045584503565237, "y1998": 1.0018840754492748, "y2006": 0.92232873520447411, "y2007": 0.91284090705064902, "y2004": 0.93694786512729977, "y2005": 0.94308212820743131, "y2002": 0.96834820215592055, "y2003": 0.95335147249088092, "y2000": 0.99127006477048718, "y2001": 0.97925917470464008, "id": 8, "y2008": 0.89689832627117483, "y2009": 0.88928857608264111}, {"neighbors": [12, 6, 4, 3, 14], "y1995": 0.87418390853652306, "y1997": 0.84425695187978567, "y1996": 0.86416601430334228, "y1999": 0.83903043942542854, "y1998": 0.8404493987171674, "y2006": 0.87204140839730271, "y2007": 0.86633032299764789, "y2004": 0.86981997840756087, "y2005": 0.86837929279319737, "y2002": 0.86107306112852877, "y2003": 0.85007719735663123, "y2000": 0.85787080050645603, "y2001": 0.86036185149249467, "id": 9, "y2008": 0.84946077011565357, "y2009": 0.83287145944123797}, {"neighbors": [0, 7, 21, 23, 22], "y1995": 1.1419611801631209, "y1997": 1.1489271154554144, "y1996": 1.146602624490825, "y1999": 1.1443662376135306, "y1998": 1.1490959392942743, "y2006": 1.1049125811637337, "y2007": 1.1105984164317646, "y2004": 1.1119989015058092, "y2005": 1.1025779214946556, "y2002": 1.1259666377127024, "y2003": 1.1221399558345004, "y2000": 1.144501826035474, "y2001": 1.1234975172649961, "id": 10, "y2008": 1.1050979494645479, "y2009": 1.1002009697391872}, {"neighbors": [8, 13, 18, 17, 2], "y1995": 0.97282462974938089, "y1997": 0.96252588061647382, "y1996": 0.96700147279313231, "y1999": 0.96057686787383312, "y1998": 0.96538780087103548, "y2006": 0.91010201260822066, "y2007": 0.89280392121658247, "y2004": 0.94103988614185807, "y2005": 0.9212251863828258, "y2002": 0.94804194711420009, "y2003": 0.9543028555845573, "y2000": 0.95831051250950716, "y2001": 0.94480908623936988, "id": 11, "y2008": 0.89298242828382146, "y2009": 0.89165384824292859}, {"neighbors": [33, 9, 6, 25, 31], "y1995": 0.94325467991401402, "y1997": 0.96455242154753429, "y1996": 0.96436902092427723, "y1999": 0.94117647058823528, "y1998": 0.95243008993884537, "y2006": 0.9346681464882507, "y2007": 0.94281559150403071, "y2004": 0.96918424441756057, "y2005": 0.94781280876672958, "y2002": 0.95388717527096822, "y2003": 0.94597005193649519, "y2000": 0.94809269652332606, "y2001": 0.93539181553564288, "id": 12, "y2008": 0.965203150896216, "y2009": 0.967154410723015}, {"neighbors": [18, 17, 11, 8, 19], "y1995": 0.97478408425654373, "y1997": 0.98712808751954773, "y1996": 0.98169225257738801, "y1999": 0.985598971191053, "y1998": 0.98474769442356791, "y2006": 0.98416665248276058, "y2007": 0.98423613480079708, "y2004": 0.97399471186978948, "y2005": 0.96910087128357136, "y2002": 0.9820996926750224, "y2003": 0.98776529543110569, "y2000": 0.98687072733199255, "y2001": 0.99237486444837619, "id": 13, "y2008": 0.99823861244053191, "y2009": 0.99545704236827348}, {"neighbors": [4, 31, 3, 29, 12], "y1995": 0.85570268988941878, "y1997": 0.85986131704895119, "y1996": 0.85575915188345031, "y1999": 0.85380119644969055, "y1998": 0.85693406055397725, "y2006": 0.82803647591954255, "y2007": 0.81987360180979219, "y2004": 0.83998883284341452, "y2005": 0.83478547261894065, "y2002": 0.85472102128186755, "y2003": 0.84564834502399988, "y2000": 0.86191535266765262, "y2001": 0.84981450830432048, "id": 14, "y2008": 0.82265395167873867, "y2009": 0.83994039782937002}, {"neighbors": [19, 8, 17, 16, 13], "y1995": 0.87022046646521634, "y1997": 0.85961813213722393, "y1996": 0.85996258309339635, "y1999": 0.8394713575455558, "y1998": 0.85689572413110093, "y2006": 0.94202108334913126, "y2007": 0.94222309998743192, "y2004": 0.86763340229291142, "y2005": 0.89179316746010362, "y2002": 0.86776297543511893, "y2003": 0.86720209304280604, "y2000": 0.82785596604704892, "y2001": 0.86008789452656809, "id": 15, "y2008": 0.93902708112840494, "y2009": 0.94479183757120588}, {"neighbors": [28, 26, 15, 19, 32], "y1995": 0.90134907329491731, "y1997": 0.90403990934606904, "y1996": 0.904077381347274, "y1999": 0.90399237579083946, "y1998": 0.90201769385650832, "y2006": 0.91108803862404764, "y2007": 0.90543476309316473, "y2004": 0.94338264626469681, "y2005": 0.91981795862151561, "y2002": 0.93695966482853577, "y2003": 0.94242697007039, "y2000": 0.90906631602055099, "y2001": 0.92693339421265908, "id": 16, "y2008": 0.91737137682250491, "y2009": 0.94793657442067902}, {"neighbors": [13, 18, 11, 19, 8], "y1995": 1.1977611005602815, "y1997": 1.1843915817489725, "y1996": 1.1822256425225894, "y1999": 1.1928672308275252, "y1998": 1.1826786457339149, "y2006": 1.2392938410349985, "y2007": 1.2341867605077472, "y2004": 1.2385704217423759, "y2005": 1.2441989281116201, "y2002": 1.2262477774195681, "y2003": 1.2239707531714479, "y2000": 1.2017286912636342, "y2001": 1.2132869128474402, "id": 17, "y2008": 1.2362673914436095, "y2009": 1.2675439750795283}, {"neighbors": [13, 17, 11, 8, 19], "y1995": 1.2491967813733067, "y1997": 1.2699116090397236, "y1996": 1.2575477330927329, "y1999": 1.3062566740535762, "y1998": 1.2802065055312271, "y2006": 1.3210776560048689, "y2007": 1.329362443219563, "y2004": 1.3054484140490119, "y2005": 1.3030330249408666, "y2002": 1.3257518058685978, "y2003": 1.3079549159235695, "y2000": 1.3479002255103918, "y2001": 1.3439986302151703, "id": 18, "y2008": 1.3300124123891741, "y2009": 1.3328846185074705}, {"neighbors": [26, 17, 28, 15, 16], "y1995": 1.0676800411188558, "y1997": 1.0363730321443168, "y1996": 1.0379927554499979, "y1999": 1.0329609259280523, "y1998": 1.027684488045026, "y2006": 0.94241549375546196, "y2007": 0.92754546923532677, "y2004": 0.99614160423102482, "y2005": 0.97356208269708677, "y2002": 1.0274762326434594, "y2003": 1.0316273366809443, "y2000": 1.0505901631347052, "y2001": 1.0340505678899605, "id": 19, "y2008": 0.92549226593721745, "y2009": 0.92138101880290568}, {"neighbors": [30, 25, 24, 37, 47], "y1995": 1.0947561397632881, "y1997": 1.1165429913770684, "y1996": 1.1152679554712275, "y1999": 1.1314326394231322, "y1998": 1.1310394841195361, "y2006": 1.1090538904302065, "y2007": 1.1057776900012568, "y2004": 1.1402994437897009, "y2005": 1.1197940058085571, "y2002": 1.133670175399079, "y2003": 1.139822558851451, "y2000": 1.1388962186541665, "y2001": 1.1244221220249986, "id": 20, "y2008": 1.1116682481010467, "y2009": 1.0998515545336902}, {"neighbors": [23, 22, 7, 10, 34], "y1995": 0.76530058421804126, "y1997": 0.76542450966153397, "y1996": 0.76612841163904621, "y1999": 0.76014283909933289, "y1998": 0.7672268310234307, "y2006": 0.76842416021983684, "y2007": 0.77487117798086069, "y2004": 0.76533287692895391, "y2005": 0.78205934309410463, "y2002": 0.76156903267949927, "y2003": 0.76651951668098528, "y2000": 0.74480073263159763, "y2001": 0.76098396210261965, "id": 21, "y2008": 0.77768682781054099, "y2009": 0.78801192267396702}, {"neighbors": [21, 34, 5, 7, 29], "y1995": 0.98391336093764348, "y1997": 0.98295341320156315, "y1996": 0.98075815675295552, "y1999": 0.96913802803963667, "y1998": 0.97386015032669815, "y2006": 0.93965462091114671, "y2007": 0.93069644684632924, "y2004": 0.9635616201227476, "y2005": 0.94745351657235244, "y2002": 0.97209860866113018, "y2003": 0.97441312580606143, "y2000": 0.97370819354423843, "y2001": 0.96419154157867693, "id": 22, "y2008": 0.94020973488297466, "y2009": 0.94358232339833159}, {"neighbors": [21, 10, 22, 34, 7], "y1995": 0.83561828119099946, "y1997": 0.81738501913392403, "y1996": 0.82298088022609361, "y1999": 0.80904800725677739, "y1998": 0.81748588141426259, "y2006": 0.87170334233473346, "y2007": 0.8786379876833581, "y2004": 0.85954307066870839, "y2005": 0.86790023653402792, "y2002": 0.83451612857812574, "y2003": 0.85175031934895873, "y2000": 0.80071489233375537, "y2001": 0.83358255807316928, "id": 23, "y2008": 0.87497981001981484, "y2009": 0.87888675419592222}, {"neighbors": [27, 20, 30, 32, 47], "y1995": 0.98845573274970278, "y1997": 0.99665282989553183, "y1996": 1.0209242772035507, "y1999": 0.99386618594343845, "y1998": 0.99141823200404444, "y2006": 0.97906748937234156, "y2007": 0.9932312332800689, "y2004": 1.0111665058188304, "y2005": 0.9998802359352077, "y2002": 0.99669586934394627, "y2003": 1.0255909749831356, "y2000": 0.98733194819247994, "y2001": 0.99644997431653437, "id": 24, "y2008": 1.0020493856497013, "y2009": 0.99602148231561483}, {"neighbors": [20, 33, 6, 30, 12], "y1995": 1.1493091345649815, "y1997": 1.143009615936718, "y1996": 1.1524194939429724, "y1999": 1.1398468268822266, "y1998": 1.1426554202510555, "y2006": 1.0889107875354573, "y2007": 1.0860369499254896, "y2004": 1.0856975145267398, "y2005": 1.1244348633192611, "y2002": 1.0423089214343333, "y2003": 1.0557727834721793, "y2000": 1.0831239730629278, "y2001": 1.0519262599166714, "id": 25, "y2008": 1.0599731384290745, "y2009": 1.0216094265950888}, {"neighbors": [28, 19, 16, 32, 17], "y1995": 1.1136826889802023, "y1997": 1.1189343096757198, "y1996": 1.1057147027213501, "y1999": 1.1432271991365353, "y1998": 1.1377866945457653, "y2006": 1.1268023587150906, "y2007": 1.1235793669317915, "y2004": 1.1482023546040769, "y2005": 1.1238659840114973, "y2002": 1.1600919581655105, "y2003": 1.1446778932605579, "y2000": 1.1825702862895446, "y2001": 1.1622624279436105, "id": 26, "y2008": 1.115925801617498, "y2009": 1.1257082797404696}, {"neighbors": [32, 24, 36, 16, 28], "y1995": 1.303794309231981, "y1997": 1.3120636604057812, "y1996": 1.3075218596998686, "y1999": 1.3062566740535762, "y1998": 1.3153226688859194, "y2006": 1.2865667454509278, "y2007": 1.2973409698906584, "y2004": 1.2683078569016086, "y2005": 1.2617743046198988, "y2002": 1.2920319347677043, "y2003": 1.2718351646774422, "y2000": 1.3121023910310281, "y2001": 1.2998915587009874, "id": 27, "y2008": 1.2939020510829768, "y2009": 1.2934544564717687}, {"neighbors": [26, 16, 19, 32, 27], "y1995": 0.83953719020532513, "y1997": 0.82006005316292385, "y1996": 0.82701447583159737, "y1999": 0.80294863992835086, "y1998": 0.8118887636743225, "y2006": 0.8389109342655191, "y2007": 0.84349246817602375, "y2004": 0.83108634437662732, "y2005": 0.84373783646216949, "y2002": 0.82596790474192727, "y2003": 0.82435704751379402, "y2000": 0.78772975118465016, "y2001": 0.82848010958278628, "id": 28, "y2008": 0.85637272428125033, "y2009": 0.86539395164519117}, {"neighbors": [5, 39, 22, 14, 31], "y1995": 1.2345008725695852, "y1997": 1.2353793515744536, "y1996": 1.2426021999018138, "y1999": 1.2452262575926329, "y1998": 1.2358129278404693, "y2006": 1.2365329681906834, "y2007": 1.2796200872578414, "y2004": 1.1967443443492951, "y2005": 1.2153657295128597, "y2002": 1.1937780418204111, "y2003": 1.1835533748469893, "y2000": 1.2256766974812463, "y2001": 1.2112664802237314, "id": 29, "y2008": 1.2796839248335934, "y2009": 1.2590773758694083}, {"neighbors": [37, 20, 24, 25, 27], "y1995": 0.97696620404861145, "y1997": 0.98035944080980575, "y1996": 0.9740071914763756, "y1999": 0.95543282313901556, "y1998": 0.97581530789338955, "y2006": 0.92100464312607799, "y2007": 0.9147530387633086, "y2004": 0.9298883479571457, "y2005": 0.93442917452618346, "y2002": 0.93679072759857129, "y2003": 0.92540049332494034, "y2000": 0.96480308308405971, "y2001": 0.9468637634838194, "id": 30, "y2008": 0.90249622070947177, "y2009": 0.90213630440783921}, {"neighbors": [35, 14, 33, 12, 4], "y1995": 0.84986885942491119, "y1997": 0.84295996568390696, "y1996": 0.89868510090623221, "y1999": 0.85659367787716301, "y1998": 0.87280533962476625, "y2006": 0.92562487931452408, "y2007": 0.96635366357254426, "y2004": 0.92698332540482575, "y2005": 0.94745351657235244, "y2002": 0.90448992922937876, "y2003": 0.95495898185605821, "y2000": 0.88937573313051443, "y2001": 0.89440100450887505, "id": 31, "y2008": 1.025203118044723, "y2009": 1.0394296020754366}, {"neighbors": [36, 27, 28, 16, 26], "y1995": 1.0192280751235561, "y1997": 1.0097442843101825, "y1996": 1.0025820319237864, "y1999": 0.99765073314119712, "y1998": 1.0030341681355639, "y2006": 0.94779637858468868, "y2007": 0.93759089358493275, "y2004": 0.97583768316642261, "y2005": 0.96101679691008712, "y2002": 0.99747298060178258, "y2003": 0.99550758543481688, "y2000": 1.0075901875261932, "y2001": 0.99192968437874551, "id": 32, "y2008": 0.93353431146829191, "y2009": 0.94121705123804411}, {"neighbors": [44, 25, 12, 35, 31], "y1995": 0.86367410708901315, "y1997": 0.85544345781923936, "y1996": 0.85558931627900803, "y1999": 0.84336613427334628, "y1998": 0.85103025143102673, "y2006": 0.89455097373003656, "y2007": 0.88283929116469462, "y2004": 0.85951183386707053, "y2005": 0.87194227372077004, "y2002": 0.84667960913556228, "y2003": 0.84374557883664714, "y2000": 0.83434853662160158, "y2001": 0.85813595114434105, "id": 33, "y2008": 0.90349490610221961, "y2009": 0.9060067497610369}, {"neighbors": [22, 39, 21, 29, 23], "y1995": 1.0094753356447226, "y1997": 1.0069881886439402, "y1996": 1.0041105523637666, "y1999": 0.99291086334982948, "y1998": 0.99513686502304577, "y2006": 0.96382634438484593, "y2007": 0.95011400973122428, "y2004": 0.975119236728752, "y2005": 0.96134614808826613, "y2002": 0.99291167539274383, "y2003": 0.98983209318633369, "y2000": 1.0058162611397035, "y2001": 0.98850522230466298, "id": 34, "y2008": 0.94346860300667812, "y2009": 0.9463776450423077}, {"neighbors": [31, 38, 44, 33, 14], "y1995": 1.0571257066143651, "y1997": 1.0575301194645879, "y1996": 1.0545941857842291, "y1999": 1.0510385688532684, "y1998": 1.0488078570498685, "y2006": 1.0247627521629479, "y2007": 1.0234752320591773, "y2004": 1.0329697933620496, "y2005": 1.0219168238570018, "y2002": 1.0420048344203974, "y2003": 1.0402553971511816, "y2000": 1.0480002306104303, "y2001": 1.030249414987729, "id": 35, "y2008": 1.0251768368501768, "y2009": 1.0435957064486703}, {"neighbors": [32, 43, 27, 28, 42], "y1995": 1.070841888164505, "y1997": 1.0793762307014196, "y1996": 1.0666949726007404, "y1999": 1.0794043012481198, "y1998": 1.0738798776109699, "y2006": 1.087727556316465, "y2007": 1.0885954360198933, "y2004": 1.1032213602455734, "y2005": 1.0916793915985508, "y2002": 1.0938347765734742, "y2003": 1.1052447043433509, "y2000": 1.0531800956589803, "y2001": 1.0745277096056161, "id": 36, "y2008": 1.0917733838297285, "y2009": 1.1096083021948762}, {"neighbors": [30, 40, 20, 42, 41], "y1995": 0.8671922185905101, "y1997": 0.86675155621455668, "y1996": 0.86628895935887062, "y1999": 0.86511809486628932, "y1998": 0.86425631732335095, "y2006": 0.84488343470424199, "y2007": 0.83374328958471722, "y2004": 0.84517414191529749, "y2005": 0.84843857600526962, "y2002": 0.85411284725399572, "y2003": 0.84886336375435456, "y2000": 0.86287327291635718, "y2001": 0.8516979624450659, "id": 37, "y2008": 0.82812044014430564, "y2009": 0.82878598934619596}, {"neighbors": [35, 31, 45, 39, 44], "y1995": 0.8838921149583755, "y1997": 0.90282398478743275, "y1996": 0.92288667453925455, "y1999": 0.92023285988219217, "y1998": 0.91229185518735723, "y2006": 0.93869676706720051, "y2007": 0.96947770975097391, "y2004": 0.99223700402629367, "y2005": 0.97984969609868555, "y2002": 0.93682451504456421, "y2003": 0.98655146182882891, "y2000": 0.92652175166361039, "y2001": 0.94278865361566122, "id": 38, "y2008": 1.0036262573224608, "y2009": 0.98102350657197357}, {"neighbors": [29, 34, 38, 22, 35], "y1995": 0.970820642185237, "y1997": 0.94534081352108112, "y1996": 0.95320232993219844, "y1999": 0.93967000034446724, "y1998": 0.94215592860799646, "y2006": 0.91035556215514757, "y2007": 0.90430364292511256, "y2004": 0.92879505989982103, "y2005": 0.9211054223180335, "y2002": 0.93412151936513388, "y2003": 0.93501274320242933, "y2000": 0.93092108910210503, "y2001": 0.92662519262599163, "id": 39, "y2008": 0.89994694483851023, "y2009": 0.9007386435858511}, {"neighbors": [41, 37, 42, 30, 45], "y1995": 0.95861858457245008, "y1997": 0.98254810501535106, "y1996": 0.95774543235102894, "y1999": 0.98684823919808018, "y1998": 0.98919471947721893, "y2006": 0.97163003599581876, "y2007": 0.97007020126757271, "y2004": 0.9493488753775261, "y2005": 0.97152609359561659, "y2002": 0.95601578436851964, "y2003": 0.94905384541254967, "y2000": 0.98882204635713133, "y2001": 0.97662233890759653, "id": 40, "y2008": 0.97158948117089283, "y2009": 0.95884908006927827}, {"neighbors": [40, 45, 44, 37, 42], "y1995": 0.83980438854721107, "y1997": 0.85746999875029983, "y1996": 0.84726737166133714, "y1999": 0.85567509846023126, "y1998": 0.85467221160427542, "y2006": 0.8333891885768886, "y2007": 0.83511679264592342, "y2004": 0.81743586206088703, "y2005": 0.83550405700769481, "y2002": 0.84502402428191115, "y2003": 0.82645665158259707, "y2000": 0.84818516243622177, "y2001": 0.85265681182580899, "id": 41, "y2008": 0.82136617314598481, "y2009": 0.80921873783836296}, {"neighbors": [43, 40, 46, 37, 36], "y1995": 0.95118156405662746, "y1997": 0.94688098462868708, "y1996": 0.9466212002600608, "y1999": 0.95124410099780687, "y1998": 0.95085829660091703, "y2006": 0.96895367966714574, "y2007": 0.9700163384024274, "y2004": 0.97583768316642261, "y2005": 0.95571723704302525, "y2002": 0.96804411514198463, "y2003": 0.97136213864358201, "y2000": 0.95440787445922959, "y2001": 0.96364362764682376, "id": 42, "y2008": 0.97082732652905901, "y2009": 0.9878236640328002}, {"neighbors": [36, 42, 32, 27, 46], "y1995": 1.0891004415267045, "y1997": 1.0849289528525252, "y1996": 1.0824896838138709, "y1999": 1.0945424900391545, "y1998": 1.0865692335830259, "y2006": 1.1450297539219478, "y2007": 1.1447474729339102, "y2004": 1.1334273474293739, "y2005": 1.1468606844516303, "y2002": 1.1229257675733433, "y2003": 1.1302103089739621, "y2000": 1.1055818811158884, "y2001": 1.1214085953998059, "id": 43, "y2008": 1.1408403740471014, "y2009": 1.1614292649793569}, {"neighbors": [33, 41, 45, 35, 40], "y1995": 1.0633603345917013, "y1997": 1.0869149629649646, "y1996": 1.0736582323828732, "y1999": 1.1166986255755473, "y1998": 1.0976484597942771, "y2006": 1.0839806574563229, "y2007": 1.0983176831786272, "y2004": 1.0927882684985315, "y2005": 1.0700320368873319, "y2002": 1.0881584856466706, "y2003": 1.0804431312806149, "y2000": 1.1185670222649935, "y2001": 1.0976428286056732, "id": 44, "y2008": 1.0929823187788443, "y2009": 1.0917612486217978}, {"neighbors": [41, 44, 40, 35, 33], "y1995": 0.79772064970019041, "y1997": 0.7858115114280021, "y1996": 0.78829195801876151, "y1999": 0.77035744221561353, "y1998": 0.77615921755360906, "y2006": 0.79949806580432425, "y2007": 0.80172181625581262, "y2004": 0.79603865293896003, "y2005": 0.78966436120841943, "y2002": 0.81437881076636964, "y2003": 0.80788827809912023, "y2000": 0.77751193519846906, "y2001": 0.79902973574567659, "id": 45, "y2008": 0.82168154748053679, "y2009": 0.85587910681858015}, {"neighbors": [42, 43, 40, 36, 37], "y1995": 1.0052446952315301, "y1997": 1.0047589936197736, "y1996": 1.0000769567582628, "y1999": 1.0063956091903872, "y1998": 1.0061394183885444, "y2006": 0.97292595590233411, "y2007": 0.96519561197191939, "y2004": 0.99030032232474696, "y2005": 0.97682565346267858, "y2002": 1.0081498135355325, "y2003": 1.0057431552702318, "y2000": 1.0016297948675874, "y2001": 0.99860738542320637, "id": 46, "y2008": 0.9617340332161447, "y2009": 0.95890283625473927}, {"neighbors": [20, 6, 24, 25, 30], "y1995": 0.95808418788867844, "y1997": 0.9654440995572009, "y1996": 0.93825679674127938, "y1999": 0.96987289157318213, "y1998": 0.95561201303757848, "y2006": 1.1704973973021624, "y2007": 1.1702515395802287, "y2004": 1.0533361880299275, "y2005": 1.0983262971945267, "y2002": 1.0078119390756035, "y2003": 1.0348423554112989, "y2000": 0.96608031008233231, "y2001": 0.99727184521431422, "id": 47, "y2008": 1.1873055260044207, "y2009": 1.1424264534188653}] diff --git a/release/python/0.2.0/crankshaft/test/helper.py b/release/python/0.2.0/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.2.0/crankshaft/test/mock_plpy.py b/release/python/0.2.0/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..a982ebe --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/mock_plpy.py @@ -0,0 +1,52 @@ +import re + +class MockCursor: + def __init__(self, data): + self.cursor_pos = 0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos : self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def debug(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def cursor(self, query): + data = self.execute(query) + return MockCursor(data) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.2.0/crankshaft/test/test_cluster_kmeans.py b/release/python/0.2.0/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.2.0/crankshaft/test/test_clustering_moran.py b/release/python/0.2.0/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..2b683cf --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,88 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.params_markov = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.2.0/crankshaft/test/test_pysal_utils.py b/release/python/0.2.0/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..171fdbc --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,142 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + self.params_array = {"id_col": "cartodb_id", + "time_cols": ["_2013_dec", "_2014_jan", "_2014_feb"], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " + + ans_array = "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params_array), ans_array) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"andy\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" IS NOT NULL AND " \ + "idx_replace.\"jay_z\" <> 0" + + ans_array = "idx_replace.\"_2013_dec\" IS NOT NULL AND " \ + "idx_replace.\"_2014_jan\" IS NOT NULL AND " \ + "idx_replace.\"_2014_feb\" IS NOT NULL" + + self.assertEqual(pu.query_attr_where(self.params), ans) + self.assertEqual(pu.query_attr_where(self.params_array), ans_array) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + ans_array = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"_2013_dec\"::numeric As attr1, " \ + "i.\"_2014_jan\"::numeric As attr2, " \ + "i.\"_2014_feb\"::numeric As attr3, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"_2013_dec\" IS NOT NULL AND " \ + "j.\"_2014_jan\" IS NOT NULL AND " \ + "j.\"_2014_feb\" IS NOT NULL " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"_2013_dec\" IS NOT NULL AND " \ + "i.\"_2014_jan\" IS NOT NULL AND " \ + "i.\"_2014_feb\" IS NOT NULL "\ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + self.assertEqual(pu.knn(self.params_array), ans_array) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/release/python/0.2.0/crankshaft/test/test_segmentation.py b/release/python/0.2.0/crankshaft/test/test_segmentation.py new file mode 100644 index 0000000..d02e8b1 --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/test_segmentation.py @@ -0,0 +1,64 @@ +import unittest +import numpy as np +from helper import plpy, fixture_file +import crankshaft.segmentation as segmentation +import json + +class SegmentationTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + + def generate_random_data(self,n_samples,random_state, row_type=False): + x1 = random_state.uniform(size=n_samples) + x2 = random_state.uniform(size=n_samples) + x3 = random_state.randint(0, 4, size=n_samples) + + y = x1+x2*x2+x3 + cartodb_id = range(len(x1)) + + if row_type: + return [ {'features': vals} for vals in zip(x1,x2,x3)], y + else: + return [dict( zip(['x1','x2','x3','target', 'cartodb_id'],[x1,x2,x3,y,cartodb_id]))] + + def test_replace_nan_with_mean(self): + test_array = np.array([1.2, np.nan, 3.2, np.nan, np.nan]) + + def test_create_and_predict_segment(self): + n_samples = 1000 + + random_state_train = np.random.RandomState(13) + random_state_test = np.random.RandomState(134) + training_data = self.generate_random_data(n_samples, random_state_train) + test_data, test_y = self.generate_random_data(n_samples, random_state_test, row_type=True) + + + ids = [{'cartodb_ids': range(len(test_data))}] + rows = [{'x1': 0,'x2':0,'x3':0,'y':0,'cartodb_id':0}] + + plpy._define_result('select \* from \(select \* from training\) a limit 1',rows) + plpy._define_result('.*from \(select \* from training\) as a' ,training_data) + plpy._define_result('select array_agg\(cartodb\_id order by cartodb\_id\) as cartodb_ids from \(.*\) a',ids) + plpy._define_result('.*select \* from test.*' ,test_data) + + model_parameters = {'n_estimators': 1200, + 'max_depth': 3, + 'subsample' : 0.5, + 'learning_rate': 0.01, + 'min_samples_leaf': 1} + + result = segmentation.create_and_predict_segment( + 'select * from training', + 'target', + 'select * from test', + model_parameters) + + prediction = [r[1] for r in result] + + accuracy =np.sqrt(np.mean( np.square( np.array(prediction) - np.array(test_y)))) + + self.assertEqual(len(result),len(test_data)) + self.assertTrue( result[0][2] < 0.01) + self.assertTrue( accuracy < 0.5*np.mean(test_y) ) diff --git a/release/python/0.2.0/crankshaft/test/test_space_time_dynamics.py b/release/python/0.2.0/crankshaft/test/test_space_time_dynamics.py new file mode 100644 index 0000000..54ffc9d --- /dev/null +++ b/release/python/0.2.0/crankshaft/test/test_space_time_dynamics.py @@ -0,0 +1,324 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.space_time_dynamics as std +from crankshaft import random_seeds +import json + +class SpaceTimeTests(unittest.TestCase): + """Testing class for Markov Functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "time_cols": ['dec_2013', 'jan_2014', 'feb_2014'], + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors_markov.json')).read()) + self.markov_data = json.loads(open(fixture_file('markov.json')).read()) + + self.time_data = np.array([i * np.ones(10, dtype=float) for i in range(10)]).T + + self.transition_matrix = np.array([ + [[ 0.96341463, 0.0304878 , 0.00609756, 0. , 0. ], + [ 0.06040268, 0.83221477, 0.10738255, 0. , 0. ], + [ 0. , 0.14 , 0.74 , 0.12 , 0. ], + [ 0. , 0.03571429, 0.32142857, 0.57142857, 0.07142857], + [ 0. , 0. , 0. , 0.16666667, 0.83333333]], + [[ 0.79831933, 0.16806723, 0.03361345, 0. , 0. ], + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0.00537634, 0.06989247, 0.8655914 , 0.05913978, 0. ], + [ 0. , 0. , 0.06372549, 0.90196078, 0.03431373], + [ 0. , 0. , 0. , 0.19444444, 0.80555556]], + [[ 0.84693878, 0.15306122, 0. , 0. , 0. ], + [ 0.08133971, 0.78947368, 0.1291866 , 0. , 0. ], + [ 0.00518135, 0.0984456 , 0.79274611, 0.0984456 , 0.00518135], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0. , 0. , 0. , 0.10204082, 0.89795918]], + [[ 0.8852459 , 0.09836066, 0. , 0.01639344, 0. ], + [ 0.03875969, 0.81395349, 0.13953488, 0. , 0.00775194], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0.02339181, 0.12865497, 0.75438596, 0.09356725], + [ 0. , 0. , 0. , 0.09661836, 0.90338164]], + [[ 0.33333333, 0.66666667, 0. , 0. , 0. ], + [ 0.0483871 , 0.77419355, 0.16129032, 0.01612903, 0. ], + [ 0.01149425, 0.16091954, 0.74712644, 0.08045977, 0. ], + [ 0. , 0.01036269, 0.06217617, 0.89637306, 0.03108808], + [ 0. , 0. , 0. , 0.02352941, 0.97647059]]] + ) + + def test_spatial_markov(self): + """Test Spatial Markov.""" + data = [ { 'id': d['id'], + 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'], + 'neighbors': d['neighbors'] } for d in self.neighbors_data] + print(str(data[0])) + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + + result = std.spatial_markov_trend('subquery', ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009'], 5, 'knn', 5, 0, 'the_geom', 'cartodb_id') + + self.assertTrue(result != None) + result = [(row[0], row[1], row[2], row[3], row[4]) for row in result] + print result[0] + expected = self.markov_data + for ([res_trend, res_up, res_down, res_vol, res_id], + [exp_trend, exp_up, exp_down, exp_vol, exp_id] + ) in zip(result, expected): + self.assertAlmostEqual(res_trend, exp_trend) + + def test_get_time_data(self): + """Test get_time_data""" + data = [ { 'attr1': d['y1995'], + 'attr2': d['y1996'], + 'attr3': d['y1997'], + 'attr4': d['y1998'], + 'attr5': d['y1999'], + 'attr6': d['y2000'], + 'attr7': d['y2001'], + 'attr8': d['y2002'], + 'attr9': d['y2003'], + 'attr10': d['y2004'], + 'attr11': d['y2005'], + 'attr12': d['y2006'], + 'attr13': d['y2007'], + 'attr14': d['y2008'], + 'attr15': d['y2009'] } for d in self.neighbors_data] + + result = std.get_time_data(data, ['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']) + + ## expected was prepared from PySAL example: + ### f = ps.open(ps.examples.get_path("usjoin.csv")) + ### pci = np.array([f.by_col[str(y)] for y in range(1995, 2010)]).transpose() + ### rpci = pci / (pci.mean(axis = 0)) + + expected = np.array([[ 0.87654416, 0.863147, 0.85637567, 0.84811668, 0.8446154, 0.83271652 + , 0.83786314, 0.85012593, 0.85509656, 0.86416612, 0.87119375, 0.86302631 + , 0.86148267, 0.86252252, 0.86746356], + [ 0.9188951, 0.91757931, 0.92333258, 0.92517289, 0.92552388, 0.90746978 + , 0.89830489, 0.89431991, 0.88924794, 0.89815176, 0.91832091, 0.91706054 + , 0.90139505, 0.87897455, 0.86216858], + [ 0.82591007, 0.82548596, 0.81989793, 0.81503235, 0.81731522, 0.78964559 + , 0.80584442, 0.8084998, 0.82258551, 0.82668196, 0.82373724, 0.81814804 + , 0.83675961, 0.83574199, 0.84647177], + [ 1.09088176, 1.08537689, 1.08456418, 1.08415404, 1.09898841, 1.14506948 + , 1.12151133, 1.11160697, 1.10888621, 1.11399806, 1.12168029, 1.13164797 + , 1.12958508, 1.11371818, 1.09936775], + [ 1.10731446, 1.11373944, 1.13283638, 1.14472559, 1.15910025, 1.16898201 + , 1.17212488, 1.14752303, 1.11843284, 1.11024964, 1.11943471, 1.11736468 + , 1.10863242, 1.09642516, 1.07762337], + [ 1.42269757, 1.42118434, 1.44273502, 1.43577571, 1.44400684, 1.44184737 + , 1.44782832, 1.41978227, 1.39092208, 1.4059372, 1.40788646, 1.44052766 + , 1.45241216, 1.43306098, 1.4174431 ], + [ 1.13073885, 1.13110513, 1.11074708, 1.13364636, 1.13088149, 1.10888138 + , 1.11856629, 1.13062931, 1.11944984, 1.12446239, 1.11671008, 1.10880034 + , 1.08401709, 1.06959206, 1.07875225], + [ 1.04706124, 1.04516831, 1.04253372, 1.03239987, 1.02072545, 0.99854316 + , 0.9880258, 0.99669587, 0.99327676, 1.01400905, 1.03176742, 1.040511 + , 1.01749645, 0.9936394, 0.98279746], + [ 0.98996986, 1.00143564, 0.99491, 1.00188408, 1.00455845, 0.99127006 + , 0.97925917, 0.9683482, 0.95335147, 0.93694787, 0.94308213, 0.92232874 + , 0.91284091, 0.89689833, 0.88928858], + [ 0.87418391, 0.86416601, 0.84425695, 0.8404494, 0.83903044, 0.8578708 + , 0.86036185, 0.86107306, 0.8500772, 0.86981998, 0.86837929, 0.87204141 + , 0.86633032, 0.84946077, 0.83287146], + [ 1.14196118, 1.14660262, 1.14892712, 1.14909594, 1.14436624, 1.14450183 + , 1.12349752, 1.12596664, 1.12213996, 1.1119989, 1.10257792, 1.10491258 + , 1.11059842, 1.10509795, 1.10020097], + [ 0.97282463, 0.96700147, 0.96252588, 0.9653878, 0.96057687, 0.95831051 + , 0.94480909, 0.94804195, 0.95430286, 0.94103989, 0.92122519, 0.91010201 + , 0.89280392, 0.89298243, 0.89165385], + [ 0.94325468, 0.96436902, 0.96455242, 0.95243009, 0.94117647, 0.9480927 + , 0.93539182, 0.95388718, 0.94597005, 0.96918424, 0.94781281, 0.93466815 + , 0.94281559, 0.96520315, 0.96715441], + [ 0.97478408, 0.98169225, 0.98712809, 0.98474769, 0.98559897, 0.98687073 + , 0.99237486, 0.98209969, 0.9877653, 0.97399471, 0.96910087, 0.98416665 + , 0.98423613, 0.99823861, 0.99545704], + [ 0.85570269, 0.85575915, 0.85986132, 0.85693406, 0.8538012, 0.86191535 + , 0.84981451, 0.85472102, 0.84564835, 0.83998883, 0.83478547, 0.82803648 + , 0.8198736, 0.82265395, 0.8399404 ], + [ 0.87022047, 0.85996258, 0.85961813, 0.85689572, 0.83947136, 0.82785597 + , 0.86008789, 0.86776298, 0.86720209, 0.8676334, 0.89179317, 0.94202108 + , 0.9422231, 0.93902708, 0.94479184], + [ 0.90134907, 0.90407738, 0.90403991, 0.90201769, 0.90399238, 0.90906632 + , 0.92693339, 0.93695966, 0.94242697, 0.94338265, 0.91981796, 0.91108804 + , 0.90543476, 0.91737138, 0.94793657], + [ 1.1977611, 1.18222564, 1.18439158, 1.18267865, 1.19286723, 1.20172869 + , 1.21328691, 1.22624778, 1.22397075, 1.23857042, 1.24419893, 1.23929384 + , 1.23418676, 1.23626739, 1.26754398], + [ 1.24919678, 1.25754773, 1.26991161, 1.28020651, 1.30625667, 1.34790023 + , 1.34399863, 1.32575181, 1.30795492, 1.30544841, 1.30303302, 1.32107766 + , 1.32936244, 1.33001241, 1.33288462], + [ 1.06768004, 1.03799276, 1.03637303, 1.02768449, 1.03296093, 1.05059016 + , 1.03405057, 1.02747623, 1.03162734, 0.9961416, 0.97356208, 0.94241549 + , 0.92754547, 0.92549227, 0.92138102], + [ 1.09475614, 1.11526796, 1.11654299, 1.13103948, 1.13143264, 1.13889622 + , 1.12442212, 1.13367018, 1.13982256, 1.14029944, 1.11979401, 1.10905389 + , 1.10577769, 1.11166825, 1.09985155], + [ 0.76530058, 0.76612841, 0.76542451, 0.76722683, 0.76014284, 0.74480073 + , 0.76098396, 0.76156903, 0.76651952, 0.76533288, 0.78205934, 0.76842416 + , 0.77487118, 0.77768683, 0.78801192], + [ 0.98391336, 0.98075816, 0.98295341, 0.97386015, 0.96913803, 0.97370819 + , 0.96419154, 0.97209861, 0.97441313, 0.96356162, 0.94745352, 0.93965462 + , 0.93069645, 0.94020973, 0.94358232], + [ 0.83561828, 0.82298088, 0.81738502, 0.81748588, 0.80904801, 0.80071489 + , 0.83358256, 0.83451613, 0.85175032, 0.85954307, 0.86790024, 0.87170334 + , 0.87863799, 0.87497981, 0.87888675], + [ 0.98845573, 1.02092428, 0.99665283, 0.99141823, 0.99386619, 0.98733195 + , 0.99644997, 0.99669587, 1.02559097, 1.01116651, 0.99988024, 0.97906749 + , 0.99323123, 1.00204939, 0.99602148], + [ 1.14930913, 1.15241949, 1.14300962, 1.14265542, 1.13984683, 1.08312397 + , 1.05192626, 1.04230892, 1.05577278, 1.08569751, 1.12443486, 1.08891079 + , 1.08603695, 1.05997314, 1.02160943], + [ 1.11368269, 1.1057147, 1.11893431, 1.13778669, 1.1432272, 1.18257029 + , 1.16226243, 1.16009196, 1.14467789, 1.14820235, 1.12386598, 1.12680236 + , 1.12357937, 1.1159258, 1.12570828], + [ 1.30379431, 1.30752186, 1.31206366, 1.31532267, 1.30625667, 1.31210239 + , 1.29989156, 1.29203193, 1.27183516, 1.26830786, 1.2617743, 1.28656675 + , 1.29734097, 1.29390205, 1.29345446], + [ 0.83953719, 0.82701448, 0.82006005, 0.81188876, 0.80294864, 0.78772975 + , 0.82848011, 0.8259679, 0.82435705, 0.83108634, 0.84373784, 0.83891093 + , 0.84349247, 0.85637272, 0.86539395], + [ 1.23450087, 1.2426022, 1.23537935, 1.23581293, 1.24522626, 1.2256767 + , 1.21126648, 1.19377804, 1.18355337, 1.19674434, 1.21536573, 1.23653297 + , 1.27962009, 1.27968392, 1.25907738], + [ 0.9769662, 0.97400719, 0.98035944, 0.97581531, 0.95543282, 0.96480308 + , 0.94686376, 0.93679073, 0.92540049, 0.92988835, 0.93442917, 0.92100464 + , 0.91475304, 0.90249622, 0.9021363 ], + [ 0.84986886, 0.8986851, 0.84295997, 0.87280534, 0.85659368, 0.88937573 + , 0.894401, 0.90448993, 0.95495898, 0.92698333, 0.94745352, 0.92562488 + , 0.96635366, 1.02520312, 1.0394296 ], + [ 1.01922808, 1.00258203, 1.00974428, 1.00303417, 0.99765073, 1.00759019 + , 0.99192968, 0.99747298, 0.99550759, 0.97583768, 0.9610168, 0.94779638 + , 0.93759089, 0.93353431, 0.94121705], + [ 0.86367411, 0.85558932, 0.85544346, 0.85103025, 0.84336613, 0.83434854 + , 0.85813595, 0.84667961, 0.84374558, 0.85951183, 0.87194227, 0.89455097 + , 0.88283929, 0.90349491, 0.90600675], + [ 1.00947534, 1.00411055, 1.00698819, 0.99513687, 0.99291086, 1.00581626 + , 0.98850522, 0.99291168, 0.98983209, 0.97511924, 0.96134615, 0.96382634 + , 0.95011401, 0.9434686, 0.94637765], + [ 1.05712571, 1.05459419, 1.05753012, 1.04880786, 1.05103857, 1.04800023 + , 1.03024941, 1.04200483, 1.0402554, 1.03296979, 1.02191682, 1.02476275 + , 1.02347523, 1.02517684, 1.04359571], + [ 1.07084189, 1.06669497, 1.07937623, 1.07387988, 1.0794043, 1.0531801 + , 1.07452771, 1.09383478, 1.1052447, 1.10322136, 1.09167939, 1.08772756 + , 1.08859544, 1.09177338, 1.1096083 ], + [ 0.86719222, 0.86628896, 0.86675156, 0.86425632, 0.86511809, 0.86287327 + , 0.85169796, 0.85411285, 0.84886336, 0.84517414, 0.84843858, 0.84488343 + , 0.83374329, 0.82812044, 0.82878599], + [ 0.88389211, 0.92288667, 0.90282398, 0.91229186, 0.92023286, 0.92652175 + , 0.94278865, 0.93682452, 0.98655146, 0.992237, 0.9798497, 0.93869677 + , 0.96947771, 1.00362626, 0.98102351], + [ 0.97082064, 0.95320233, 0.94534081, 0.94215593, 0.93967, 0.93092109 + , 0.92662519, 0.93412152, 0.93501274, 0.92879506, 0.92110542, 0.91035556 + , 0.90430364, 0.89994694, 0.90073864], + [ 0.95861858, 0.95774543, 0.98254811, 0.98919472, 0.98684824, 0.98882205 + , 0.97662234, 0.95601578, 0.94905385, 0.94934888, 0.97152609, 0.97163004 + , 0.9700702, 0.97158948, 0.95884908], + [ 0.83980439, 0.84726737, 0.85747, 0.85467221, 0.8556751, 0.84818516 + , 0.85265681, 0.84502402, 0.82645665, 0.81743586, 0.83550406, 0.83338919 + , 0.83511679, 0.82136617, 0.80921874], + [ 0.95118156, 0.9466212, 0.94688098, 0.9508583, 0.9512441, 0.95440787 + , 0.96364363, 0.96804412, 0.97136214, 0.97583768, 0.95571724, 0.96895368 + , 0.97001634, 0.97082733, 0.98782366], + [ 1.08910044, 1.08248968, 1.08492895, 1.08656923, 1.09454249, 1.10558188 + , 1.1214086, 1.12292577, 1.13021031, 1.13342735, 1.14686068, 1.14502975 + , 1.14474747, 1.14084037, 1.16142926], + [ 1.06336033, 1.07365823, 1.08691496, 1.09764846, 1.11669863, 1.11856702 + , 1.09764283, 1.08815849, 1.08044313, 1.09278827, 1.07003204, 1.08398066 + , 1.09831768, 1.09298232, 1.09176125], + [ 0.79772065, 0.78829196, 0.78581151, 0.77615922, 0.77035744, 0.77751194 + , 0.79902974, 0.81437881, 0.80788828, 0.79603865, 0.78966436, 0.79949807 + , 0.80172182, 0.82168155, 0.85587911], + [ 1.0052447, 1.00007696, 1.00475899, 1.00613942, 1.00639561, 1.00162979 + , 0.99860739, 1.00814981, 1.00574316, 0.99030032, 0.97682565, 0.97292596 + , 0.96519561, 0.96173403, 0.95890284], + [ 0.95808419, 0.9382568, 0.9654441, 0.95561201, 0.96987289, 0.96608031 + , 0.99727185, 1.00781194, 1.03484236, 1.05333619, 1.0983263, 1.1704974 + , 1.17025154, 1.18730553, 1.14242645]]) + + self.assertTrue(np.allclose(result, expected)) + self.assertTrue(type(result) == type(expected)) + self.assertTrue(result.shape == expected.shape) + + def test_rebin_data(self): + """Test rebin_data""" + ## sample in double the time (even case since 10 % 2 = 0): + ## (0+1)/2, (2+3)/2, (4+5)/2, (6+7)/2, (8+9)/2 + ## = 0.5, 2.5, 4.5, 6.5, 8.5 + ans_even = np.array([(i + 0.5) * np.ones(10, dtype=float) + for i in range(0, 10, 2)]).T + + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 2), ans_even)) + + ## sample in triple the time (uneven since 10 % 3 = 1): + ## (0+1+2)/3, (3+4+5)/3, (6+7+8)/3, (9)/1 + ## = 1, 4, 7, 9 + ans_odd = np.array([i * np.ones(10, dtype=float) + for i in (1, 4, 7, 9)]).T + self.assertTrue(np.array_equal(std.rebin_data(self.time_data, 3), ans_odd)) + + def test_get_prob_dist(self): + """Test get_prob_dist""" + lag_indices = np.array([1, 2, 3, 4]) + unit_indices = np.array([1, 3, 2, 4]) + answer = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + result = std.get_prob_dist(self.transition_matrix, lag_indices, unit_indices) + + self.assertTrue(np.array_equal(result, answer)) + + def test_get_prob_stats(self): + """Test get_prob_stats""" + + probs = np.array([ + [ 0.0754717 , 0.88207547, 0.04245283, 0. , 0. ], + [ 0. , 0. , 0.09411765, 0.87058824, 0.03529412], + [ 0.0049505 , 0.09405941, 0.77722772, 0.11881188, 0.0049505 ], + [ 0. , 0. , 0. , 0.02352941, 0.97647059] + ]) + unit_indices = np.array([1, 3, 2, 4]) + answer_up = np.array([0.04245283, 0.03529412, 0.12376238, 0.]) + answer_down = np.array([0.0754717, 0.09411765, 0.0990099, 0.02352941]) + answer_trend = np.array([-0.03301887 / 0.88207547, -0.05882353 / 0.87058824, 0.02475248 / 0.77722772, -0.02352941 / 0.97647059]) + answer_volatility = np.array([ 0.34221495, 0.33705421, 0.29226542, 0.38834223]) + + result = std.get_prob_stats(probs, unit_indices) + result_up = result[0] + result_down = result[1] + result_trend = result[2] + result_volatility = result[3] + + self.assertTrue(np.allclose(result_up, answer_up)) + self.assertTrue(np.allclose(result_down, answer_down)) + self.assertTrue(np.allclose(result_trend, answer_trend)) + self.assertTrue(np.allclose(result_volatility, answer_volatility)) From 9ada8d12ca6e86fa07bd4cdb408f32ed3d80091b Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 11 Aug 2016 18:52:27 +0200 Subject: [PATCH 178/183] Add upgrade file from 0.1.0 to 0.2.0 As compatibility was checked with CI tests, this is simply a copy: ``` cp release/crankshaft--0.2.0.sql release/crankshaft--0.1.0--0.2.0.sql ``` To be automated in `make release` command. --- release/crankshaft--0.1.0--0.2.0.sql | 827 +++++++++++++++++++++++++++ 1 file changed, 827 insertions(+) create mode 100644 release/crankshaft--0.1.0--0.2.0.sql diff --git a/release/crankshaft--0.1.0--0.2.0.sql b/release/crankshaft--0.1.0--0.2.0.sql new file mode 100644 index 0000000..1cb3087 --- /dev/null +++ b/release/crankshaft--0.1.0--0.2.0.sql @@ -0,0 +1,827 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.2.0'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION + CDB_PyAggS(current_state Numeric[], current_row Numeric[]) + returns NUMERIC[] as $$ + BEGIN + if array_upper(current_state,1) is null then + RAISE NOTICE 'setting state %',array_upper(current_row,1); + current_state[1] = array_upper(current_row,1); + end if; + return array_cat(current_state,current_row) ; + END + $$ LANGUAGE plpgsql; + +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_pyagg' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" + ); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment( + target NUMERIC[], + features NUMERIC[], + target_features NUMERIC[], + target_ids NUMERIC[], + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC) +AS $$ + import numpy as np + import plpy + + from crankshaft.segmentation import create_and_predict_segment_agg + model_params = {'n_estimators': n_estimators, + 'max_depth': max_depth, + 'subsample': subsample, + 'learning_rate': learning_rate, + 'min_samples_leaf': min_samples_leaf} + + def unpack2D(data): + dimension = data.pop(0) + a = np.array(data, dtype=float) + return a.reshape(len(a)/dimension, dimension) + + return create_and_predict_segment_agg(np.array(target, dtype=float), + unpack2D(features), + unpack2D(target_features), + target_ids, + model_params) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION + CDB_CreateAndPredictSegment ( + query TEXT, + variable_name TEXT, + target_table TEXT, + n_estimators INTEGER DEFAULT 1200, + max_depth INTEGER DEFAULT 3, + subsample DOUBLE PRECISION DEFAULT 0.5, + learning_rate DOUBLE PRECISION DEFAULT 0.01, + min_samples_leaf INTEGER DEFAULT 1) +RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC) +AS $$ + from crankshaft.segmentation import create_and_predict_segment + model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf} + return create_and_predict_segment(query,variable_name,target_table, model_params) +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN target_query text, + IN weight_column text, + IN source_query text, + IN pop_column text, + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_id bigint[]; + t_geom geometry[]; + t_weight numeric[]; + s_id bigint[]; + s_geom geometry[]; + s_pop numeric[]; +BEGIN + EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight; + EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop; + RETURN QUERY + SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g; +END; +$$ language plpgsql; + +CREATE OR REPLACE FUNCTION CDB_Gravity( + IN t_id bigint[], + IN t_geom geometry[], + IN t_weight numeric[], + IN s_id bigint[], + IN s_geom geometry[], + IN s_pop numeric[], + IN target bigint, + IN radius integer, + IN minval numeric DEFAULT -10e307 + ) +RETURNS TABLE( + the_geom geometry, + source_id bigint, + target_id bigint, + dist numeric, + h numeric, + hpop numeric) AS $$ +DECLARE + t_type text; + s_type text; + t_center geometry[]; + s_center geometry[]; +BEGIN + t_type := GeometryType(t_geom[1]); + s_type := GeometryType(s_geom[1]); + IF t_type = 'POINT' THEN + t_center := t_geom; + ELSE + WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp; + END IF; + IF s_type = 'POINT' THEN + s_center := s_geom; + ELSE + WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp; + END IF; + RETURN QUERY + with target0 as( + SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td + ), + source0 as( + SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp + ), + prev0 as( + SELECT + source0.sg, + source0.sd as sourc_id, + coalesce(source0.sp,0) as sp, + target.td as targ_id, + coalesce(target.tw,0) as tw, + GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance + FROM source0 + CROSS JOIN LATERAL + ( + SELECT + * + FROM target0 + WHERE tw > minval + AND ST_DWithin(geography(source0.sc), geography(tc), radius) + ) AS target + ), + deno as( + SELECT + sourc_id, + sum(tw/distance) as h_deno + FROM + prev0 + GROUP BY sourc_id + ) + SELECT + p.sg as the_geom, + p.sourc_id as source_id, + p.targ_id as target_id, + case when p.distance > 1 then p.distance else 0.0 end as dist, + 100*(p.tw/p.distance)/d.h_deno as h, + p.sp*(p.tw/p.distance)/d.h_deno as hpop + FROM + prev0 p, + deno d + WHERE + p.targ_id = target AND + p.sourc_id = d.sourc_id; +END; +$$ language plpgsql; +-- 0: nearest neighbor +-- 1: barymetric +-- 2: IDW + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN query text, + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + output numeric; +BEGIN + EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs; + SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a; + + RETURN output; +END; +$$ +language plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION CDB_SpatialInterpolation( + IN geomin geometry[], + IN colin numeric[], + IN point geometry, + IN method integer DEFAULT 1, + IN p1 numeric DEFAULT 0, + IN p2 numeric DEFAULT 0 + ) +RETURNS numeric AS +$$ +DECLARE + gs geometry[]; + vs numeric[]; + gs2 geometry[]; + vs2 numeric[]; + g geometry; + vertex geometry[]; + sg numeric; + sa numeric; + sb numeric; + sc numeric; + va numeric; + vb numeric; + vc numeric; + output numeric; +BEGIN + output := -999.999; + -- nearest + IF method = 0 THEN + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v) + SELECT a.v INTO output FROM a ORDER BY point<->a.g LIMIT 1; + RETURN output; + + -- barymetric + ELSIF method = 1 THEN + WITH a as (SELECT unnest(geomin) AS e), + b as (SELECT ST_DelaunayTriangles(ST_Collect(a.e),0.001, 0) AS t FROM a), + c as (SELECT (ST_Dump(t)).geom as v FROM b), + d as (SELECT v FROM c WHERE ST_Within(point, v)) + SELECT v INTO g FROM d; + IF g is null THEN + -- out of the realm of the input data + RETURN -888.888; + END IF; + -- vertex of the selected cell + WITH a AS (SELECT (ST_DumpPoints(g)).geom AS v) + SELECT array_agg(v) INTO vertex FROM a; + + -- retrieve the value of each vertex + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]); + WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]); + + SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc; + + output := (coalesce(sa,0) * coalesce(va,0) + coalesce(sb,0) * coalesce(vb,0) + coalesce(sc,0) * coalesce(vc,0)) / coalesce(sg); + RETURN output; + + -- IDW + -- p1: limit the number of neighbors, 0->no limit + -- p2: order of distance decay, 0-> order 1 + ELSIF method = 2 THEN + + IF p2 = 0 THEN + p2 := 1; + END IF; + + WITH a as (SELECT unnest(geomin) as g, unnest(colin) as v), + b as (SELECT a.g, a.v FROM a ORDER BY point<->a.g) + SELECT array_agg(b.g), array_agg(b.v) INTO gs, vs FROM b; + IF p1::integer>0 THEN + gs2:=gs; + vs2:=vs; + FOR i IN 1..p1 + LOOP + gs2 := gs2 || gs[i]; + vs2 := vs2 || vs[i]; + END LOOP; + ELSE + gs2:=gs; + vs2:=vs; + END IF; + + WITH a as (SELECT unnest(gs2) as g, unnest(vs2) as v), + b as ( + SELECT + (1/ST_distance(point, a.g)^p2::integer) as k, + (a.v/ST_distance(point, a.g)^p2::integer) as f + FROM a + ) + SELECT sum(b.f)/sum(b.k) INTO output FROM b; + RETURN output; + + END IF; + + RETURN -777.777; + +END; +$$ +language plpgsql IMMUTABLE; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_weightedmean' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" + ); + END IF; +END +$$ LANGUAGE plpgsql; +-- Spatial Markov + +-- input table format: +-- id | geom | date_1 | date_2 | date_3 +-- 1 | Pt1 | 12.3 | 13.1 | 14.2 +-- 2 | Pt2 | 11.0 | 13.2 | 12.5 +-- ... +-- Sample Function call: +-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate', +-- Array['date_1', 'date_2', 'date_3']) + +CREATE OR REPLACE FUNCTION + CDB_SpatialMarkovTrend ( + subquery TEXT, + time_cols TEXT[], + num_classes INT DEFAULT 7, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT) +AS $$ + + from crankshaft.space_time_dynamics import spatial_markov_trend + + ## TODO: use named parameters or a dictionary + return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- input table format: identical to above but in a predictable format +-- Sample function call: +-- SELECT cdb_spatial_markov('SELECT * FROM real_estate', +-- 'date_1') + + +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col_min text, +-- time_col_max text, +-- date_format text, -- '_YYYY_MM_DD' +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- +-- -- input table format: +-- -- id | geom | date | measurement +-- -- 1 | Pt1 | 12/3 | 13.2 +-- -- 2 | Pt2 | 11/5 | 11.3 +-- -- 3 | Pt1 | 11/13 | 12.9 +-- -- 4 | Pt3 | 12/19 | 10.1 +-- -- ... +-- +-- CREATE OR REPLACE FUNCTION +-- cdb_spatial_markov ( +-- subquery TEXT, +-- time_col text, +-- num_time_per_bin INT DEFAULT 1, +-- permutations INT DEFAULT 99, +-- geom_column TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs int DEFAULT 5) +-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +-- AS $$ +-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') +-- from crankshaft.clustering import moran_local +-- # TODO: use named parameters or a dictionary +-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; From 47f1f918d45433c87a55cd5d2375aef9fc015c5f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 11 Aug 2016 19:20:02 +0200 Subject: [PATCH 179/183] Do not check code is updated when not needed Particularly, when it is master or a detached HEAD (usually when checking out tags). --- check-up-to-date-with-master.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/check-up-to-date-with-master.sh b/check-up-to-date-with-master.sh index af4b8b4..2d8fcfe 100755 --- a/check-up-to-date-with-master.sh +++ b/check-up-to-date-with-master.sh @@ -1,5 +1,13 @@ #!/bin/bash +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +if [[ "$CURRENT_BRANCH" == "master" || "$CURRENT_BRANCH" == "HEAD" ]] +then + echo "master branch or detached HEAD" + exit 0 +fi + # Add remote-master git remote add -t master remote-master https://github.com/CartoDB/crankshaft.git From 170260b7f5953731ff13eb89880dc99b864720b6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 12 Aug 2016 18:17:13 +0200 Subject: [PATCH 180/183] Revamp the dev process --- README.md | 77 ++++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index ee72c18..683fb38 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,65 @@ -# crankshaft [![Build Status](https://travis-ci.org/CartoDB/crankshaft.svg?branch=develop)](https://travis-ci.org/CartoDB/crankshaft) +# Crankshaft [![Build Status](https://travis-ci.org/CartoDB/crankshaft.svg?branch=develop)](https://travis-ci.org/CartoDB/crankshaft) CartoDB Spatial Analysis extension for PostgreSQL. ## Code organization -* *doc* documentation -* *src* source code -* - *src/pg* contains the PostgreSQL extension source code -* - *src/py* Python module source code -* *release* reseleased versions +* `doc/` documentation +* `src/` source code + - `pg/` contains the PostgreSQL extension source code + - `py/` Python module source code +* `release` reseleased versions ## Requirements -* pip, PostgreSQL -* python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/master/src/py/README.md)) +* PostgreSQL +* plpythonu and postgis extensions +* python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/develop/src/py/README.md)) -# Working Process -- Quickstart Guide +# Development Process -We distinguish two roles regarding the development cycle of crankshaft: +We distinguish two roles: * *developers* will implement new functionality and bugfixes into - the codebase and will request for new releases of the extension. -* A *release manager* will attend these requests and will handle - the release process. The release process is sequential: - no concurrent releases will ever be in the works. + the codebase. +* A *release manager* will handle the release process. -We use the default `develop` branch as the basis for development. -The `master` branch is used to merge and tag releases to be -deployed in production. +We use the branch `develop` as the main integration branch for development. The `master` is reserved to handle releases. -Developers shall create a new topic branch from `develop` for any new feature -or bugfix and commit their changes to it and eventually merge back into -the `develop` branch. When a new release is required a Pull Request -will be open against the `develop` branch. +The process is as follows: + +1. Create a new **topic branch** from `develop` for any new feature +or bugfix and commit their changes to it: +```shell +git fetch && git checkout -b my-cool-feature origin/develop +``` +1. Code, commit, push, repeat. +1. Write some **tests** for your feature or bugfix. +1. Create a pull request and mention relevant people for a **peer review**. +1. Address the comments and improvements you get from the peer review. +1. Mention `@CartoDB/dataservices` in the PR to get it merged into `develop`. + +In order for a pull request to be accepted, the following criteria should be met: +* The peer review should pass and no major issue should be left unaddressed. +* CI tests must pass (travis will take care of that). -The `develop` pull requests will be handled by the release manage, -who will merge into master where new releases are prepared and tagged. -The `master` branch is the sole responsibility of the release masters -and developers must not commit or merge into it. ## Development Guidelines For a detailed description of the development process please see -the [CONTRIBUTING.md](https://github.com/CartoDB/crankshaft/blob/master/CONTRIBUTING.md) guide. +the [CONTRIBUTING.md](https://github.com/CartoDB/crankshaft/blob/develop/CONTRIBUTING.md) guide. -Any modification to the source code (`src/pg/sql` for the SQL extension, -`src/py/crankshaft` for the Python package) shall always be done -in a topic branch created from the `develop` branch. -Tests, documentation and peer code reviewing are required for all -modifications. +## Testing -The tests (both for SQL and Python) are executed by running, -from the top directory: +The tests (both for SQL and Python) are executed by running, from the top directory: -``` +```shell sudo make install make test ``` -To request a new release, which will be handled by them -release manager, a Pull Request must be created in the `develop` -branch. - ## Release -The release and deployment process is described in the -[RELEASE.md](https://github.com/CartoDB/crankshaft/blob/master/RELEASE.md) guide and it is the responsibility of the designated -release manager. +The release process is described in the +[RELEASE.md](https://github.com/CartoDB/crankshaft/blob/develop/RELEASE.md) guide and is the responsibility of the designated *release manager*. From 065dc476b4021d19602621eaf0b95acc8f84ee80 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 12 Aug 2016 18:34:13 +0200 Subject: [PATCH 181/183] Revamp the dev process --- CONTRIBUTING.md | 86 ++++++++++++++----------------------------------- README.md | 1 + 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42385dc..ed694bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,7 @@ # Development process -Please read the Working Process/Quickstart Guide in [README.md](https://github.com/CartoDB/crankshaft/blob/master/README.md) first. - For any modification of crankshaft, such as adding new features, -refactoring or bug-fixing, topic branch must be created out of the `develop` -branch and be used for the development process. +refactoring or bugfixing, a topic branch must be created out of the `develop`. Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. @@ -14,80 +11,47 @@ Take into account: (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to detect any bugs that are being fixed. * Add or modify the corresponding documentation files in the `doc` folder. - Since we expect to have highly technical functions here, an extense - background explanation would be of great help to users of this extension. -* Convention: snake case(i.e. `snake_case` and not `CamelCase`) - shall be used for all function names. - Prefix function names intended for public use with `cdb_` - and private functions (to be used only internally inside - the extension) with `_cdb_`. +* Naming conventions for function names: + - use `CamelCase` + - prefix "public" functions with `CDB_`. E.g: `CDB_SpatialMarkovTrend` + - prefix "private" functions with an underscore. E.g: `_CDB_MyObscureInternalImplementationDetail` Once the code is ready to be tested, update the local development installation with `sudo make install`. This will update the 'dev' version of the extension in `src/pg/` and make it available to PostgreSQL. -It will also install the python package (crankshaft) in a virtual -environment `env/dev`. - -The version number of the Python package, defined in -`src/pg/crankshaft/setup.py` will be overridden when -the package is released and always match the extension version number, -but for development it shall be kept as '0.0.0'. Run the tests with `make test`. -To use the python extension for custom tests, activate the virtual -environment with: - -``` -source envs/dev/bin/activate -``` - Update extension in a working database with: -* `ALTER EXTENSION crankshaft UPDATE TO 'current';` - `ALTER EXTENSION crankshaft UPDATE TO 'dev';` - -Note: we keep the current development version install as 'dev' always; -we update through the 'current' alias to allow changing the extension -contents but not the version identifier. This will fail if the -changes involve incompatible function changes such as a different -return type; in that case the offending function (or the whole extension) -should be dropped manually before the update. +```sql +ALTER EXTENSION crankshaft UPDATE TO 'current'; +ALTER EXTENSION crankshaft UPDATE TO 'dev'; +``` If the extension has not previously been installed in a database, it can be installed directly with: - -* `CREATE EXTENSION IF NOT EXISTS plpythonu;` - `CREATE EXTENSION IF NOT EXISTS postgis;` - `CREATE EXTENSION crankshaft WITH VERSION 'dev';` - -Note: the development extension uses the development python virtual -environment automatically. - -Before proceeding to the release process peer code reviewing of the code is -a must. +```sql +CREATE EXTENSION IF NOT EXISTS plpythonu; +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION crankshaft WITH VERSION 'dev'; +``` Once the feature or bugfix is completed and all the tests are passing -a Pull-Request shall be created on the topic branch, reviewed by a peer -and then merged back into the `develop` branch when all CI tests pass. +a pull request shall be created, reviewed by a peer +and then merged back into the `develop` branch once all the CI tests pass. -When the changes in the `develop` branch are to be released in a new -version of the extension, a PR must be created on the `develop` branch. -The release manage will take hold of the PR at this moment to proceed -to the release process for a new revision of the extension. +## Relevant development targets in the Makefile -## Relevant development tasks available in the Makefile +```shell +# Show a short description of the available targets +make help -``` -* `make help` show a short description of the available targets - -* `sudo make install` will generate the extension scripts for the development - version ('dev'/'current') and install the python package into the - development virtual environment `envs/dev`. - Intended for use by developers. - -* `make test` will run the tests for the installed development extension. - Intended for use by developers. +# Generate the extension scripts and install the python package. +sudo make install + +# Run the tests against the installed extension. +make test ``` diff --git a/README.md b/README.md index 683fb38..9dad032 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ git fetch && git checkout -b my-cool-feature origin/develop ``` 1. Code, commit, push, repeat. 1. Write some **tests** for your feature or bugfix. +1. Update the [NEWS.md](https://github.com/CartoDB/crankshaft/blob/develop/NEWS.md) doc. 1. Create a pull request and mention relevant people for a **peer review**. 1. Address the comments and improvements you get from the peer review. 1. Mention `@CartoDB/dataservices` in the PR to get it merged into `develop`. From 8953bf92ee85067a96aaa547a986fd7da145dc8d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Fri, 12 Aug 2016 18:56:03 +0200 Subject: [PATCH 182/183] Update release process --- RELEASE.md | 107 +++++++++++++++-------------------------------------- 1 file changed, 29 insertions(+), 78 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 0db48a2..005557a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,93 +1,44 @@ # Release & Deployment Process -Please read the Working Process/Quickstart Guide in README.md -and the Development guidelines in CONTRIBUTING.md. - The release process of a new version of the extension shall be performed by the designated *Release Manager*. -Note that we expect to gradually automate more of this process. - -Having checked PR to be released it shall be -merged back into the `master` branch to prepare the new release. - -The version number in `pg/cranckshaft.control` must first be updated. -To do so [Semantic Versioning 2.0](http://semver.org/) is in order. - -Thew `NEWS.md` will be updated. - -We now will explain the process for the case of backwards-compatible -releases (updating the minor or patch version numbers). - -TODO: document the complex case of major releases. - -The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql` and -also to copy the python package to `release/python/X.Y.Z/crankshaft`. - -``` +## Release steps +1. Make sure `develop` branch passes all the tests. +1. Merge `develop` into `master` +1. Update the version number in `src/pg/crankshaft.control`. +1. Generate the next release files with this command: +```shell make release ``` +1. Generate an upgrade path from the previous to the next release by copying the generated release file. E.g: +```shell +cp release/cranckshaft--X.Y.Z.sql release/cranckshaft--A.B.C--X.Y.Z.sql +``` +NOTE: you can rely on this thanks to the compatibility checks. TODO: automate this step [#94](https://github.com/CartoDB/crankshaft/issues/94) +1. Commit and push the generated files. +1. Tag the release: +``` +git tag -a X.Y.Z -m "Release X.Y.Z" +git push origin X.Y.Z +``` +1. Deploy and test in staging -Then, the release manager shall produce upgrade and downgrade scripts -to migrate to/from the previous release. In the case of minor/patch -releases this simply consist in extracting the functions that have changed -and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` -file. + +## Some remarks +* Version numbers shall follow [Semantic Versioning 2.0](http://semver.org/). +* CI tests will take care of **forward compatibility** of the extension at postgres level. +* **Major version changes** (breaking forward compatibility) are a major event and are out of the scope of this doc. They **shall be avoided as much as we can**. +* We will go forward, never backwards. **Generating upgrade paths automatically is easy** and we'll rely on the CI checks for that. + +## Deploy commands The new release can be deployed for staging/smoke tests with this command: - -``` +```shell sudo make deploy ``` -This will copy the current 'X.Y.Z' released version of the extension to -PostgreSQL. The corresponding Python extension will be installed in a -virtual environment in `envs/X.Y.Z`. - -It can be activated with: - -``` -source envs/X.Y.Z/bin/activate -``` - -But note that this is needed only for using the package directly; -the 'X.Y.Z' version of the extension will automatically use the -python package from this virtual environment. - -The `sudo make deploy` operation can be also used for installing -the new version after it has been released. - -To install a specific version 'X.Y.Z' different from the current one -(which must be present in `releases/`) you can: - -``` +To install a specific version 'X.Y.Z' different from the default one: +```shell sudo make deploy RELEASE_VERSION=X.Y.Z ``` - -TODO: testing procedure for the new release. - -TODO: procedure for staging deployment. - -TODO: procedure for merging to master, tagging and deploying -in production. - -## Relevant release & deployment tasks available in the Makefile - -``` -* `make help` show a short description of the available targets - -* `make release` will generate a new release (version number defined in - `src/pg/crankshaft.control`) into `release/`. - Intended for use by the release manager. - -* `sudo make deploy` will install the current release X.Y.Z from the - `release/` files into PostgreSQL and a Python virtual environment - `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. - -* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version - previously generated in `release/` - into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. -``` From 7914fcdb87fd6962597c46db9056fd2546f3ab89 Mon Sep 17 00:00:00 2001 From: abelvm Date: Mon, 15 Aug 2016 16:32:25 -0400 Subject: [PATCH 183/183] fix barycenter method --- src/pg/sql/08_interpolation.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/sql/08_interpolation.sql b/src/pg/sql/08_interpolation.sql index 76fad01..0937f09 100644 --- a/src/pg/sql/08_interpolation.sql +++ b/src/pg/sql/08_interpolation.sql @@ -74,11 +74,11 @@ BEGIN SELECT array_agg(v) INTO vertex FROM a; -- retrieve the value of each vertex - WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c) SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]); - WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c) SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]); - WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c) + WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c) SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]); SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc;