add full tests
This commit is contained in:
abelvm
@@ -4,3 +4,4 @@ results/
|
||||
crankshaft--dev.sql
|
||||
crankshaft--dev--current.sql
|
||||
crankshaft--current--dev.sql
|
||||
crankshaft--*--dev.sql
|
||||
|
||||
+6
-10
@@ -7,22 +7,19 @@ include ../../Makefile.global
|
||||
# requires sudo. In additionof the current development version
|
||||
# named 'dev', an alias 'current' is generating for ease of
|
||||
# update (upgrade to 'current', then to 'dev').
|
||||
# the python module is installed in a virtualenv in envs/dev/
|
||||
# * test runs the tests for the currently generated Development
|
||||
# extension.
|
||||
|
||||
DATA = $(EXTENSION)--dev.sql \
|
||||
$(EXTENSION)--current--dev.sql \
|
||||
$(EXTENSION)--dev--current.sql
|
||||
DATA = \
|
||||
$(EXTENSION)--dev.sql \
|
||||
$(EXTENSION)--current--dev.sql \
|
||||
$(EXTENSION)--dev--current.sql \
|
||||
$(EXTENSION)--$(RELEASE_VERSION)--dev.sql
|
||||
|
||||
SOURCES_DATA_DIR = sql
|
||||
SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
|
||||
|
||||
VIRTUALENV_PATH = $(realpath ../../envs)
|
||||
ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH))
|
||||
|
||||
REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \
|
||||
-e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g'
|
||||
REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
|
||||
|
||||
$(DATA): $(SOURCES_DATA)
|
||||
$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@
|
||||
@@ -54,7 +51,6 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
|
||||
$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql
|
||||
|
||||
# Install the current relese into the PostgreSQL extensions directory
|
||||
# and the Python package in a virtual environment envs/X.Y.Z
|
||||
deploy:
|
||||
$(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
|
||||
$(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/'
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB Spatial Analysis extension'
|
||||
default_version = '0.0.2'
|
||||
requires = 'plpythonu, postgis, cartodb'
|
||||
default_version = '0.2.0'
|
||||
requires = 'plpythonu, postgis'
|
||||
superuser = true
|
||||
schema = cdb_crankshaft
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path()
|
||||
RETURNS text
|
||||
AS $$
|
||||
BEGIN
|
||||
-- RETURN '/opt/virtualenvs/crankshaft';
|
||||
RETURN '@@VIRTUALENV_PATH@@';
|
||||
END;
|
||||
$$ language plpgsql IMMUTABLE STRICT;
|
||||
|
||||
-- Use the crankshaft python module
|
||||
CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py()
|
||||
RETURNS VOID
|
||||
AS $$
|
||||
import os
|
||||
# plpy.notice('%',str(os.environ))
|
||||
# activate virtualenv
|
||||
crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version']
|
||||
base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path']
|
||||
default_venv_path = os.path.join(base_path, crankshaft_version)
|
||||
venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path)
|
||||
activate_path = venv_path + '/bin/activate_this.py'
|
||||
exec(open(activate_path).read(), dict(__file__=activate_path))
|
||||
$$ LANGUAGE plpythonu;
|
||||
@@ -4,7 +4,6 @@
|
||||
CREATE OR REPLACE FUNCTION
|
||||
_cdb_random_seeds (seed_value INTEGER) RETURNS VOID
|
||||
AS $$
|
||||
plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
from crankshaft import random_seeds
|
||||
random_seeds.set_random_seeds(seed_value)
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
CREATE OR REPLACE FUNCTION
|
||||
CDB_PyAggS(current_state Numeric[], current_row Numeric[])
|
||||
returns NUMERIC[] as $$
|
||||
BEGIN
|
||||
if array_upper(current_state,1) is null then
|
||||
RAISE NOTICE 'setting state %',array_upper(current_row,1);
|
||||
current_state[1] = array_upper(current_row,1);
|
||||
end if;
|
||||
return array_cat(current_state,current_row) ;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Create aggregate if it did not exist
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT *
|
||||
FROM pg_catalog.pg_proc p
|
||||
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname = 'cdb_crankshaft'
|
||||
AND p.proname = 'cdb_pyagg'
|
||||
AND p.proisagg)
|
||||
THEN
|
||||
CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) (
|
||||
SFUNC = CDB_PyAggS,
|
||||
STYPE = Numeric[],
|
||||
INITCOND = "{}"
|
||||
);
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,53 @@
|
||||
|
||||
CREATE OR REPLACE FUNCTION
|
||||
CDB_CreateAndPredictSegment(
|
||||
target NUMERIC[],
|
||||
features NUMERIC[],
|
||||
target_features NUMERIC[],
|
||||
target_ids NUMERIC[],
|
||||
n_estimators INTEGER DEFAULT 1200,
|
||||
max_depth INTEGER DEFAULT 3,
|
||||
subsample DOUBLE PRECISION DEFAULT 0.5,
|
||||
learning_rate DOUBLE PRECISION DEFAULT 0.01,
|
||||
min_samples_leaf INTEGER DEFAULT 1)
|
||||
RETURNS TABLE(cartodb_id NUMERIC, prediction NUMERIC, accuracy NUMERIC)
|
||||
AS $$
|
||||
import numpy as np
|
||||
import plpy
|
||||
|
||||
from crankshaft.segmentation import create_and_predict_segment_agg
|
||||
model_params = {'n_estimators': n_estimators,
|
||||
'max_depth': max_depth,
|
||||
'subsample': subsample,
|
||||
'learning_rate': learning_rate,
|
||||
'min_samples_leaf': min_samples_leaf}
|
||||
|
||||
def unpack2D(data):
|
||||
dimension = data.pop(0)
|
||||
a = np.array(data, dtype=float)
|
||||
return a.reshape(len(a)/dimension, dimension)
|
||||
|
||||
return create_and_predict_segment_agg(np.array(target, dtype=float),
|
||||
unpack2D(features),
|
||||
unpack2D(target_features),
|
||||
target_ids,
|
||||
model_params)
|
||||
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION
|
||||
CDB_CreateAndPredictSegment (
|
||||
query TEXT,
|
||||
variable_name TEXT,
|
||||
target_table TEXT,
|
||||
n_estimators INTEGER DEFAULT 1200,
|
||||
max_depth INTEGER DEFAULT 3,
|
||||
subsample DOUBLE PRECISION DEFAULT 0.5,
|
||||
learning_rate DOUBLE PRECISION DEFAULT 0.01,
|
||||
min_samples_leaf INTEGER DEFAULT 1)
|
||||
RETURNS TABLE (cartodb_id TEXT, prediction NUMERIC, accuracy NUMERIC)
|
||||
AS $$
|
||||
from crankshaft.segmentation import create_and_predict_segment
|
||||
model_params = {'n_estimators': n_estimators, 'max_depth':max_depth, 'subsample' : subsample, 'learning_rate': learning_rate, 'min_samples_leaf' : min_samples_leaf}
|
||||
return create_and_predict_segment(query,variable_name,target_table, model_params)
|
||||
$$ LANGUAGE plpythonu;
|
||||
@@ -0,0 +1,115 @@
|
||||
CREATE OR REPLACE FUNCTION CDB_Gravity(
|
||||
IN target_query text,
|
||||
IN weight_column text,
|
||||
IN source_query text,
|
||||
IN pop_column text,
|
||||
IN target bigint,
|
||||
IN radius integer,
|
||||
IN minval numeric DEFAULT -10e307
|
||||
)
|
||||
RETURNS TABLE(
|
||||
the_geom geometry,
|
||||
source_id bigint,
|
||||
target_id bigint,
|
||||
dist numeric,
|
||||
h numeric,
|
||||
hpop numeric) AS $$
|
||||
DECLARE
|
||||
t_id bigint[];
|
||||
t_geom geometry[];
|
||||
t_weight numeric[];
|
||||
s_id bigint[];
|
||||
s_geom geometry[];
|
||||
s_pop numeric[];
|
||||
BEGIN
|
||||
EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight;
|
||||
EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop;
|
||||
RETURN QUERY
|
||||
SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g;
|
||||
END;
|
||||
$$ language plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION CDB_Gravity(
|
||||
IN t_id bigint[],
|
||||
IN t_geom geometry[],
|
||||
IN t_weight numeric[],
|
||||
IN s_id bigint[],
|
||||
IN s_geom geometry[],
|
||||
IN s_pop numeric[],
|
||||
IN target bigint,
|
||||
IN radius integer,
|
||||
IN minval numeric DEFAULT -10e307
|
||||
)
|
||||
RETURNS TABLE(
|
||||
the_geom geometry,
|
||||
source_id bigint,
|
||||
target_id bigint,
|
||||
dist numeric,
|
||||
h numeric,
|
||||
hpop numeric) AS $$
|
||||
DECLARE
|
||||
t_type text;
|
||||
s_type text;
|
||||
t_center geometry[];
|
||||
s_center geometry[];
|
||||
BEGIN
|
||||
t_type := GeometryType(t_geom[1]);
|
||||
s_type := GeometryType(s_geom[1]);
|
||||
IF t_type = 'POINT' THEN
|
||||
t_center := t_geom;
|
||||
ELSE
|
||||
WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp;
|
||||
END IF;
|
||||
IF s_type = 'POINT' THEN
|
||||
s_center := s_geom;
|
||||
ELSE
|
||||
WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp;
|
||||
END IF;
|
||||
RETURN QUERY
|
||||
with target0 as(
|
||||
SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td
|
||||
),
|
||||
source0 as(
|
||||
SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp
|
||||
),
|
||||
prev0 as(
|
||||
SELECT
|
||||
source0.sg,
|
||||
source0.sd as sourc_id,
|
||||
coalesce(source0.sp,0) as sp,
|
||||
target.td as targ_id,
|
||||
coalesce(target.tw,0) as tw,
|
||||
GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance
|
||||
FROM source0
|
||||
CROSS JOIN LATERAL
|
||||
(
|
||||
SELECT
|
||||
*
|
||||
FROM target0
|
||||
WHERE tw > minval
|
||||
AND ST_DWithin(geography(source0.sc), geography(tc), radius)
|
||||
) AS target
|
||||
),
|
||||
deno as(
|
||||
SELECT
|
||||
sourc_id,
|
||||
sum(tw/distance) as h_deno
|
||||
FROM
|
||||
prev0
|
||||
GROUP BY sourc_id
|
||||
)
|
||||
SELECT
|
||||
p.sg as the_geom,
|
||||
p.sourc_id as source_id,
|
||||
p.targ_id as target_id,
|
||||
case when p.distance > 1 then p.distance else 0.0 end as dist,
|
||||
100*(p.tw/p.distance)/d.h_deno as h,
|
||||
p.sp*(p.tw/p.distance)/d.h_deno as hpop
|
||||
FROM
|
||||
prev0 p,
|
||||
deno d
|
||||
WHERE
|
||||
p.targ_id = target AND
|
||||
p.sourc_id = d.sourc_id;
|
||||
END;
|
||||
$$ language plpgsql;
|
||||
@@ -14,9 +14,12 @@ $$
|
||||
DECLARE
|
||||
gs geometry[];
|
||||
vs numeric[];
|
||||
output numeric;
|
||||
BEGIN
|
||||
EXECUTE 'WITH a AS('||query||') SELECT array_agg(the_geom), array_agg(attrib) FROM a' INTO gs, vs;
|
||||
RETURN QUERY SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) FROM a;
|
||||
SELECT CDB_SpatialInterpolation(gs, vs, point, method, p1,p2) INTO output FROM a;
|
||||
|
||||
RETURN output;
|
||||
END;
|
||||
$$
|
||||
language plpgsql IMMUTABLE;
|
||||
@@ -71,11 +74,11 @@ BEGIN
|
||||
SELECT array_agg(v) INTO vertex FROM a;
|
||||
|
||||
-- retrieve the value of each vertex
|
||||
WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c)
|
||||
WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c)
|
||||
SELECT c INTO va FROM a WHERE ST_Equals(geo, vertex[1]);
|
||||
WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c)
|
||||
WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c)
|
||||
SELECT c INTO vb FROM a WHERE ST_Equals(geo, vertex[2]);
|
||||
WITH a AS(SELECT unnest(vertex) as geo, unnest(colin) as c)
|
||||
WITH a AS(SELECT unnest(geomin) as geo, unnest(colin) as c)
|
||||
SELECT c INTO vc FROM a WHERE ST_Equals(geo, vertex[3]);
|
||||
|
||||
SELECT ST_area(g), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[2], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point, vertex[1], vertex[3], point]))), ST_area(ST_MakePolygon(ST_MakeLine(ARRAY[point,vertex[1],vertex[2], point]))) INTO sg, sa, sb, sc;
|
||||
|
||||
@@ -10,7 +10,6 @@ CREATE OR REPLACE FUNCTION
|
||||
id_col TEXT DEFAULT 'cartodb_id')
|
||||
RETURNS TABLE (moran NUMERIC, significance NUMERIC)
|
||||
AS $$
|
||||
plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
from crankshaft.clustering import moran_local
|
||||
# TODO: use named parameters or a dictionary
|
||||
return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
|
||||
@@ -28,7 +27,6 @@ CREATE OR REPLACE FUNCTION
|
||||
id_col TEXT)
|
||||
RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
|
||||
AS $$
|
||||
plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
from crankshaft.clustering import moran_local
|
||||
# TODO: use named parameters or a dictionary
|
||||
return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
|
||||
@@ -122,7 +120,6 @@ CREATE OR REPLACE FUNCTION
|
||||
id_col TEXT DEFAULT 'cartodb_id')
|
||||
RETURNS TABLE (moran FLOAT, significance FLOAT)
|
||||
AS $$
|
||||
plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
from crankshaft.clustering import moran_local
|
||||
# TODO: use named parameters or a dictionary
|
||||
return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
|
||||
@@ -143,7 +140,6 @@ CREATE OR REPLACE FUNCTION
|
||||
RETURNS
|
||||
TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
|
||||
AS $$
|
||||
plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
from crankshaft.clustering import moran_local_rate
|
||||
# TODO: use named parameters or a dictionary
|
||||
return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20)
|
||||
RETURNS table (cartodb_id integer, cluster_no integer) as $$
|
||||
|
||||
from crankshaft.clustering import kmeans
|
||||
return kmeans(query,no_clusters,no_init)
|
||||
|
||||
$$ language plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC)
|
||||
RETURNS Numeric[] AS
|
||||
$$
|
||||
DECLARE
|
||||
newX NUMERIC;
|
||||
newY NUMERIC;
|
||||
newW NUMERIC;
|
||||
BEGIN
|
||||
IF weight IS NULL OR the_geom IS NULL THEN
|
||||
newX = state[1];
|
||||
newY = state[2];
|
||||
newW = state[3];
|
||||
ELSE
|
||||
newX = state[1] + ST_X(the_geom)*weight;
|
||||
newY = state[2] + ST_Y(the_geom)*weight;
|
||||
newW = state[3] + weight;
|
||||
END IF;
|
||||
RETURN Array[newX,newY,newW];
|
||||
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[])
|
||||
RETURNS GEOMETRY AS
|
||||
$$
|
||||
BEGIN
|
||||
IF state[3] = 0 THEN
|
||||
RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326);
|
||||
ELSE
|
||||
RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326);
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Create aggregate if it did not exist
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT *
|
||||
FROM pg_catalog.pg_proc p
|
||||
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname = 'cdb_crankshaft'
|
||||
AND p.proname = 'cdb_weightedmean'
|
||||
AND p.proisagg)
|
||||
THEN
|
||||
CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) (
|
||||
SFUNC = CDB_WeightedMeanS,
|
||||
FINALFUNC = CDB_WeightedMeanF,
|
||||
STYPE = Numeric[],
|
||||
INITCOND = "{0.0,0.0,0.0}"
|
||||
);
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,81 @@
|
||||
-- Spatial Markov
|
||||
|
||||
-- input table format:
|
||||
-- id | geom | date_1 | date_2 | date_3
|
||||
-- 1 | Pt1 | 12.3 | 13.1 | 14.2
|
||||
-- 2 | Pt2 | 11.0 | 13.2 | 12.5
|
||||
-- ...
|
||||
-- Sample Function call:
|
||||
-- SELECT CDB_SpatialMarkov('SELECT * FROM real_estate',
|
||||
-- Array['date_1', 'date_2', 'date_3'])
|
||||
|
||||
CREATE OR REPLACE FUNCTION
|
||||
CDB_SpatialMarkovTrend (
|
||||
subquery TEXT,
|
||||
time_cols TEXT[],
|
||||
num_classes INT DEFAULT 7,
|
||||
w_type TEXT DEFAULT 'knn',
|
||||
num_ngbrs INT DEFAULT 5,
|
||||
permutations INT DEFAULT 99,
|
||||
geom_col TEXT DEFAULT 'the_geom',
|
||||
id_col TEXT DEFAULT 'cartodb_id')
|
||||
RETURNS TABLE (trend NUMERIC, trend_up NUMERIC, trend_down NUMERIC, volatility NUMERIC, rowid INT)
|
||||
AS $$
|
||||
|
||||
from crankshaft.space_time_dynamics import spatial_markov_trend
|
||||
|
||||
## TODO: use named parameters or a dictionary
|
||||
return spatial_markov_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
-- input table format: identical to above but in a predictable format
|
||||
-- Sample function call:
|
||||
-- SELECT cdb_spatial_markov('SELECT * FROM real_estate',
|
||||
-- 'date_1')
|
||||
|
||||
|
||||
-- CREATE OR REPLACE FUNCTION
|
||||
-- cdb_spatial_markov (
|
||||
-- subquery TEXT,
|
||||
-- time_col_min text,
|
||||
-- time_col_max text,
|
||||
-- date_format text, -- '_YYYY_MM_DD'
|
||||
-- num_time_per_bin INT DEFAULT 1,
|
||||
-- permutations INT DEFAULT 99,
|
||||
-- geom_column TEXT DEFAULT 'the_geom',
|
||||
-- id_col TEXT DEFAULT 'cartodb_id',
|
||||
-- w_type TEXT DEFAULT 'knn',
|
||||
-- num_ngbrs int DEFAULT 5)
|
||||
-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT)
|
||||
-- AS $$
|
||||
-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
-- from crankshaft.clustering import moran_local
|
||||
-- # TODO: use named parameters or a dictionary
|
||||
-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
|
||||
-- $$ LANGUAGE plpythonu;
|
||||
--
|
||||
-- -- input table format:
|
||||
-- -- id | geom | date | measurement
|
||||
-- -- 1 | Pt1 | 12/3 | 13.2
|
||||
-- -- 2 | Pt2 | 11/5 | 11.3
|
||||
-- -- 3 | Pt1 | 11/13 | 12.9
|
||||
-- -- 4 | Pt3 | 12/19 | 10.1
|
||||
-- -- ...
|
||||
--
|
||||
-- CREATE OR REPLACE FUNCTION
|
||||
-- cdb_spatial_markov (
|
||||
-- subquery TEXT,
|
||||
-- time_col text,
|
||||
-- num_time_per_bin INT DEFAULT 1,
|
||||
-- permutations INT DEFAULT 99,
|
||||
-- geom_column TEXT DEFAULT 'the_geom',
|
||||
-- id_col TEXT DEFAULT 'cartodb_id',
|
||||
-- w_type TEXT DEFAULT 'knn',
|
||||
-- num_ngbrs int DEFAULT 5)
|
||||
-- RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT)
|
||||
-- AS $$
|
||||
-- plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
|
||||
-- from crankshaft.clustering import moran_local
|
||||
-- # TODO: use named parameters or a dictionary
|
||||
-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
|
||||
-- $$ LANGUAGE plpythonu;
|
||||
@@ -1,6 +1,18 @@
|
||||
-- Install dependencies
|
||||
CREATE EXTENSION plpythonu;
|
||||
CREATE EXTENSION postgis;
|
||||
CREATE EXTENSION cartodb;
|
||||
CREATE EXTENSION postgis VERSION '2.2.2';
|
||||
-- Create role publicuser if it does not exist
|
||||
DO
|
||||
$$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT *
|
||||
FROM pg_catalog.pg_user
|
||||
WHERE usename = 'publicuser') THEN
|
||||
|
||||
CREATE ROLE publicuser LOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
-- Install the extension
|
||||
CREATE EXTENSION crankshaft VERSION 'dev';
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
\pset format unaligned
|
||||
\set ECHO all
|
||||
SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
|
||||
clusters
|
||||
2
|
||||
(1 row)
|
||||
SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p;
|
||||
clusters
|
||||
52
|
||||
(1 row)
|
||||
@@ -0,0 +1,10 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
_cdb_random_seeds
|
||||
|
||||
(1 row)
|
||||
cartodb_id|trend_test|trend_up_test|trend_down_test|volatility_test
|
||||
1|t|t|t|t
|
||||
2|t|t|t|t
|
||||
3|t|t|t|t
|
||||
(3 rows)
|
||||
@@ -0,0 +1,27 @@
|
||||
\pset format unaligned
|
||||
\set ECHO none
|
||||
_cdb_random_seeds
|
||||
|
||||
(1 row)
|
||||
within_tolerance
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
t
|
||||
(20 rows)
|
||||
@@ -0,0 +1,14 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
the_geom | h | hpop | dist
|
||||
--------------------------------------------+-------------------------+--------------------------+----------------
|
||||
01010000001361C3D32B650140DD24068195B34440 | 1.51078258369747945249 | 12.08626066957983561994 | 4964.714459152
|
||||
01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 | 99.955141922
|
||||
0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894711274639196 | 382.23197368267647835174 | 2488.330566505
|
||||
010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870080995954879 | 176.77079350404979774397 | 4359.370460594
|
||||
010100000052B81E85EB510140355EBA490CB24440 | 33.12290506987740864904 | 132.49162027950963459615 | 3703.664449828
|
||||
0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754279248087849 | 196.35755262837744263547 | 2512.092358644
|
||||
01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 | 2926.25725244
|
||||
0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 | 53.54905726651871279586 | 3744.515577777
|
||||
(8 rows)
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
<<<<<<< HEAD
|
||||
nn | nni | idw
|
||||
-----+--------------------------+-----------------
|
||||
200 | 780.79470198683925288365 | 341.46260750526
|
||||
=======
|
||||
cdb_spatialinterpolation
|
||||
t
|
||||
>>>>>>> ebbf68af5700f33737ac18516e2bdc22ea330b24
|
||||
(1 row)
|
||||
|
||||
|
||||
+180
File diff suppressed because one or more lines are too long
Vendored
+2005
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,20 @@
|
||||
-- Install dependencies
|
||||
CREATE EXTENSION plpythonu;
|
||||
CREATE EXTENSION postgis;
|
||||
CREATE EXTENSION cartodb;
|
||||
CREATE EXTENSION postgis VERSION '2.2.2';
|
||||
|
||||
-- Create role publicuser if it does not exist
|
||||
DO
|
||||
$$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT *
|
||||
FROM pg_catalog.pg_user
|
||||
WHERE usename = 'publicuser') THEN
|
||||
|
||||
CREATE ROLE publicuser LOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Install the extension
|
||||
CREATE EXTENSION crankshaft VERSION 'dev';
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
\pset format unaligned
|
||||
\set ECHO all
|
||||
|
||||
SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2);
|
||||
|
||||
SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p;
|
||||
@@ -0,0 +1,33 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
\pset format unaligned
|
||||
\i test/fixtures/markov_usjoin_example.sql
|
||||
|
||||
-- Areas of Interest functions perform some nondeterministic computations
|
||||
-- (to estimate the significance); we will set the seeds for the RNGs
|
||||
-- that affect those results to have repeatable results
|
||||
SELECT cdb_crankshaft._cdb_random_seeds(1234);
|
||||
|
||||
SELECT
|
||||
m1.cartodb_id,
|
||||
CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05
|
||||
WHEN m1.cartodb_id = 2 THEN abs(m2.trend - 0.15151515151515152) / 0.15151515151515152 < 0.05
|
||||
WHEN m1.cartodb_id = 3 THEN abs(m2.trend - 0.069767441860465115) / 0.069767441860465115 < 0.05
|
||||
ELSE NULL END As trend_test,
|
||||
CASE WHEN m1.cartodb_id = 1 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05
|
||||
WHEN m1.cartodb_id = 2 THEN abs(m2.trend_up - 0.13157894736842105) / 0.13157894736842105 < 0.05
|
||||
WHEN m1.cartodb_id = 3 THEN abs(m2.trend_up - 0.065217391304347824) / 0.065217391304347824 < 0.05
|
||||
ELSE NULL END As trend_up_test,
|
||||
CASE WHEN m1.cartodb_id = 1 THEN m2.trend_down = 0.0
|
||||
WHEN m1.cartodb_id = 2 THEN m2.trend_down = 0.0
|
||||
WHEN m1.cartodb_id = 3 THEN m2.trend_down = 0.0
|
||||
ELSE NULL END As trend_down_test,
|
||||
CASE WHEN m1.cartodb_id = 1 THEN abs(m2.volatility - 0.367574633389) / 0.367574633389 < 0.1
|
||||
WHEN m1.cartodb_id = 2 THEN abs(m2.volatility - 0.33807340742635211) / 0.33807340742635211 < 0.1
|
||||
WHEN m1.cartodb_id = 3 THEN abs(m2.volatility - 0.3682585596149513) / 0.3682585596149513 < 0.1
|
||||
ELSE NULL END As volatility_test
|
||||
FROM markov_usjoin_example As m1
|
||||
JOIN cdb_crankshaft.CDB_SpatialMarkovTrend('SELECT * FROM markov_usjoin_example ORDER BY cartodb_id DESC', Array['y1995', 'y1996', 'y1997', 'y1998', 'y1999', 'y2000', 'y2001', 'y2002', 'y2003', 'y2004', 'y2005', 'y2006', 'y2007', 'y2008', 'y2009']::text[], 5::int, 'knn'::text, 5::int, 0::int, 'the_geom'::text, 'cartodb_id'::text) As m2
|
||||
ON m1.cartodb_id = m2.rowid
|
||||
ORDER BY m1.cartodb_id
|
||||
LIMIT 3;
|
||||
@@ -0,0 +1,33 @@
|
||||
\pset format unaligned
|
||||
\set ECHO none
|
||||
\i test/fixtures/ml_values.sql
|
||||
SELECT cdb_crankshaft._cdb_random_seeds(1234);
|
||||
|
||||
WITH expected AS (
|
||||
SELECT generate_series(1000,1020) AS id, unnest(ARRAY[
|
||||
4.5656517130822492,
|
||||
1.7928053473230694,
|
||||
1.0283378773916563,
|
||||
2.6586517814904593,
|
||||
2.9699056242935944,
|
||||
3.9550646059951347,
|
||||
4.1662572444459745,
|
||||
3.8126334839264162,
|
||||
1.8809821053623488,
|
||||
1.6349065129019873,
|
||||
3.0391288591472954,
|
||||
3.3035970359672553,
|
||||
1.5835471589451968,
|
||||
3.7530378537263638,
|
||||
1.0833589653009252,
|
||||
3.8104965452882897,
|
||||
2.665217959294802,
|
||||
1.5850334252802472,
|
||||
3.679401198805563,
|
||||
3.5332033186588636
|
||||
]) AS expected LIMIT 20
|
||||
), prediction AS (
|
||||
SELECT cartodb_id::integer id, prediction
|
||||
FROM cdb_crankshaft.CDB_CreateAndPredictSegment('SELECT target, x1, x2, x3 FROM ml_values WHERE class = $$train$$','target','SELECT cartodb_id, target, x1, x2, x3 FROM ml_values WHERE class = $$test$$')
|
||||
LIMIT 20
|
||||
) SELECT abs(e.expected - p.prediction) <= 1e-9 AS within_tolerance FROM expected e, prediction p WHERE e.id = p.id;
|
||||
@@ -0,0 +1,24 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
|
||||
WITH t AS (
|
||||
SELECT
|
||||
ARRAY[1,2,3] AS id,
|
||||
ARRAY[7.0,8.0,3.0] AS w,
|
||||
ARRAY[ST_GeomFromText('POINT(2.1744 41.4036)'),ST_GeomFromText('POINT(2.1228 41.3809)'),ST_GeomFromText('POINT(2.1511 41.3742)')] AS g
|
||||
),
|
||||
s AS (
|
||||
SELECT
|
||||
ARRAY[10,20,30,40,50,60,70,80] AS id,
|
||||
ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS p,
|
||||
ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g
|
||||
)
|
||||
SELECT
|
||||
g.the_geom,
|
||||
g.h,
|
||||
g.hpop,
|
||||
g.dist
|
||||
FROM
|
||||
t,
|
||||
s,
|
||||
cdb_crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g;
|
||||
@@ -1,12 +1,21 @@
|
||||
SET client_min_messages TO WARNING;
|
||||
\set ECHO none
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
\pset format unaligned
|
||||
|
||||
>>>>>>> ebbf68af5700f33737ac18516e2bdc22ea330b24
|
||||
WITH a AS (
|
||||
SELECT
|
||||
ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS vals,
|
||||
ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g
|
||||
)
|
||||
<<<<<<< HEAD
|
||||
SELECT
|
||||
CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'),0) as NN,
|
||||
CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'),1) as NNI,
|
||||
CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'),2) as IDW
|
||||
FROM a;
|
||||
=======
|
||||
SELECT (cdb_crankshaft.CDB_SpatialInterpolation(g, vals, ST_GeomFromText('POINT(2.154 41.37)'), 1) - 780.79470198683925288365) / 780.79470198683925288365 < 0.001 As cdb_spatialinterpolation FROM a;
|
||||
>>>>>>> ebbf68af5700f33737ac18516e2bdc22ea330b24
|
||||
|
||||
@@ -4,7 +4,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234);
|
||||
SET ROLE test_regular_user;
|
||||
|
||||
-- Add to the search path the schema
|
||||
SET search_path TO public,cartodb,cdb_crankshaft;
|
||||
SET search_path TO public,cdb_crankshaft;
|
||||
|
||||
-- Exercise public functions
|
||||
SELECT ppoints.code, m.quads
|
||||
|
||||
Reference in New Issue
Block a user